From 6f1a0232b0deb00eb8ef432a53f52df9b35e717c Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sun, 24 May 2026 20:24:03 +0800 Subject: [PATCH 001/213] [improve][client] In cases where there is a risk of message loss, adjust the log level to error (#25854) (cherry picked from commit 09035ffddada146f3ef014777ee5e5766f01f3f2) --- .../pulsar/client/impl/ConsumerBase.java | 20 ++++++++++++------- .../pulsar/client/impl/ConsumerImpl.java | 12 +++++++++++ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java index 4e0f44669250d..a4e5bef30833b 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java @@ -331,10 +331,13 @@ protected CompletableFuture> nextPendingReceive() { protected void completePendingReceive(CompletableFuture> receivedFuture, Message message) { getInternalExecutor(message).execute(() -> { - if (!receivedFuture.complete(message)) { - log.warn("Race condition detected. receive future was already completed (cancelled={}) and message was " - + "dropped. message={}", - receivedFuture.isCancelled(), message); + if (!receivedFuture.complete(message) && getState() != State.Closing && getState() != State.Closed) { + log.error("Race condition detected, receive future was already completed and message was dropped." + + " In other words, the message was dropped internally, the client-side will encounter a" + + " crucial issue: this message will never be consumed until the consumer is restarted or" + + " the topic is unloaded. Under normal circumstances, this won't happen. It only occurs when" + + " user itself has completed the completable future object returned by" + + " \"consumer.receiveAsync()\". message={}, cancelled={}", message, receivedFuture.isCancelled()); } }); } @@ -1098,9 +1101,12 @@ protected final void notifyPendingBatchReceivedCallBack(CompletableFuture> future, Messages messages) { if (!future.complete(messages)) { - log.warn("Race condition detected. batch receive future was already completed (cancelled={}) and messages" - + " were dropped. messages={}", - future.isCancelled(), messages); + log.warn("Race condition detected, receive future was already completed and message was dropped." + + " In other words, the message was dropped internally, the client-side will encounter a" + + " crucial issue: these message will never be consumed until the consumer is restarted or" + + " the topic is unloaded. Under normal circumstances, this won't happen. It only occurs when" + + " user itself has completed the completable future object returned by" + + " \"consumer.batchReceiveAsync()\". messages={}, cancelled={}", messages, future.isCancelled()); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index cd2a205c41d18..1038d84d89cf2 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -1701,12 +1701,24 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m */ void notifyPendingReceivedCallback(final Message message, Exception exception) { if (pendingReceives.isEmpty()) { + if (getState() != State.Closing && getState() != State.Closed) { + log.error("If you received this log, it means that you encountered a bug: a message was" + + " dropped internally, the client-side will encounter a crucial issue: this message will" + + " never be consumed until the consumer is restarted or the topic is unloaded. message={}," + + " pendingReceives-size={}", message, pendingReceives.size()); + } return; } // fetch receivedCallback from queue final CompletableFuture> receivedFuture = nextPendingReceive(); if (receivedFuture == null) { + if (getState() != State.Closing && getState() != State.Closed) { + log.error("The pendingReceives pulled out a null conpletableFuture object. If you received this log," + + " it means that you encountered a bug: a message was" + + " dropped internally, the client-side will encounter a crucial issue: this message will never" + + " be consumed until the consumer is restarted or the topic is unloaded. message={}", message); + } return; } From e2ae742313a6cc9a4d76d48a5fa147560abf6c08 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Wed, 27 May 2026 11:57:11 +0800 Subject: [PATCH 002/213] [fix][client]Fix checkstyle issue of ConsumerBase.java --- .../main/java/org/apache/pulsar/client/impl/ConsumerBase.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java index a4e5bef30833b..2cc5026601334 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java @@ -337,7 +337,8 @@ protected void completePendingReceive(CompletableFuture> receivedFutu + " crucial issue: this message will never be consumed until the consumer is restarted or" + " the topic is unloaded. Under normal circumstances, this won't happen. It only occurs when" + " user itself has completed the completable future object returned by" - + " \"consumer.receiveAsync()\". message={}, cancelled={}", message, receivedFuture.isCancelled()); + + " \"consumer.receiveAsync()\". message={}, cancelled={}", message, + receivedFuture.isCancelled()); } }); } From 6891dfffd46598a7c92b8bf91b3e15aab98bcb17 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Mon, 25 May 2026 20:30:23 +0800 Subject: [PATCH 003/213] [fix][client] Fix failed to close consumer because of the error: param memorySize is a negative value (#25805) (cherry picked from commit 59d1495fef46075986a92065bd64229fe0f354ea) --- .../apache/pulsar/client/impl/ConsumerBase.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java index 2cc5026601334..65cdbf07d5b5c 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java @@ -971,13 +971,17 @@ protected boolean enqueueMessageAndCheckBatchReceive(Message message) { // synchronize redeliverUnacknowledgedMessages(). incomingQueueLock.lock(); try { - if (canEnqueueMessage(message) && incomingMessages.offer(message)) { - // After we have enqueued the messages on `incomingMessages` queue, we cannot touch the message - // instance anymore, since for pooled messages, this instance was possibly already been released - // and recycled. + if (canEnqueueMessage(message)) { INCOMING_MESSAGES_SIZE_UPDATER.addAndGet(this, messageSize); - getMemoryLimitController().ifPresent(limiter -> limiter.forceReserveMemory(messageSize)); - updateAutoScaleReceiverQueueHint(); + if (incomingMessages.offer(message)) { + // After we have enqueued the messages on `incomingMessages` queue, we cannot touch the message + // instance anymore, since for pooled messages, this instance was possibly already been released + // and recycled. + getMemoryLimitController().ifPresent(limiter -> limiter.forceReserveMemory(messageSize)); + updateAutoScaleReceiverQueueHint(); + } else { + INCOMING_MESSAGES_SIZE_UPDATER.addAndGet(this, -messageSize); + } } } finally { incomingQueueLock.unlock(); From 59a1bfd0b992d49dc29b2be272b3bd775960d69a Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 21 May 2026 09:44:56 +0300 Subject: [PATCH 004/213] [fix][sec] Upgrade commons-configuration2 to 2.15.0 to address CVE-2026-45205 (#25844) (cherry picked from commit 7220158b928b2b9dc8151400a1a596d93ecd5637) --- distribution/server/src/assemble/LICENSE.bin.txt | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 43051dc425da2..564e09229146f 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -289,7 +289,7 @@ The Apache Software License, Version 2.0 - commons-logging-commons-logging-1.3.5.jar - org.apache.commons-commons-collections4-4.5.0.jar - org.apache.commons-commons-compress-1.28.0.jar - - org.apache.commons-commons-configuration2-2.12.0.jar + - org.apache.commons-commons-configuration2-2.15.0.jar - org.apache.commons-commons-lang3-3.19.0.jar - org.apache.commons-commons-text-1.14.0.jar * Netty diff --git a/pom.xml b/pom.xml index 39d14df0835de..8aca0a9da0a9f 100644 --- a/pom.xml +++ b/pom.xml @@ -374,7 +374,7 @@ flexible messaging model and an intuitive client API. 3.33.0 9.37.4 1.11.0 - 2.12.0 + 2.15.0 2.1.10 1.10.3 From 7fd4287734f1e84d6bb02775e50d12d57ce7c8a9 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 26 May 2026 11:48:35 +0300 Subject: [PATCH 005/213] [improve][misc] Upgrade Netty to 4.1.134 (#25870) (cherry picked from commit a24a3b50b46abbacafcdef23d23ee8a03c3e25ac) --- .../server/src/assemble/LICENSE.bin.txt | 40 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 38 +++++++++--------- pom.xml | 2 +- 3 files changed, 40 insertions(+), 40 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 564e09229146f..39aa9ada72ae3 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -293,26 +293,26 @@ The Apache Software License, Version 2.0 - org.apache.commons-commons-lang3-3.19.0.jar - org.apache.commons-commons-text-1.14.0.jar * Netty - - io.netty-netty-buffer-4.1.133.Final.jar - - io.netty-netty-codec-4.1.133.Final.jar - - io.netty-netty-codec-dns-4.1.133.Final.jar - - io.netty-netty-codec-http-4.1.133.Final.jar - - io.netty-netty-codec-http2-4.1.133.Final.jar - - io.netty-netty-codec-socks-4.1.133.Final.jar - - io.netty-netty-codec-haproxy-4.1.133.Final.jar - - io.netty-netty-common-4.1.133.Final.jar - - io.netty-netty-handler-4.1.133.Final.jar - - io.netty-netty-handler-proxy-4.1.133.Final.jar - - io.netty-netty-resolver-4.1.133.Final.jar - - io.netty-netty-resolver-dns-4.1.133.Final.jar - - io.netty-netty-resolver-dns-classes-macos-4.1.133.Final.jar - - io.netty-netty-resolver-dns-native-macos-4.1.133.Final-osx-aarch_64.jar - - io.netty-netty-resolver-dns-native-macos-4.1.133.Final-osx-x86_64.jar - - io.netty-netty-transport-4.1.133.Final.jar - - io.netty-netty-transport-classes-epoll-4.1.133.Final.jar - - io.netty-netty-transport-native-epoll-4.1.133.Final-linux-aarch_64.jar - - io.netty-netty-transport-native-epoll-4.1.133.Final-linux-x86_64.jar - - io.netty-netty-transport-native-unix-common-4.1.133.Final.jar + - io.netty-netty-buffer-4.1.134.Final.jar + - io.netty-netty-codec-4.1.134.Final.jar + - io.netty-netty-codec-dns-4.1.134.Final.jar + - io.netty-netty-codec-http-4.1.134.Final.jar + - io.netty-netty-codec-http2-4.1.134.Final.jar + - io.netty-netty-codec-socks-4.1.134.Final.jar + - io.netty-netty-codec-haproxy-4.1.134.Final.jar + - io.netty-netty-common-4.1.134.Final.jar + - io.netty-netty-handler-4.1.134.Final.jar + - io.netty-netty-handler-proxy-4.1.134.Final.jar + - io.netty-netty-resolver-4.1.134.Final.jar + - io.netty-netty-resolver-dns-4.1.134.Final.jar + - io.netty-netty-resolver-dns-classes-macos-4.1.134.Final.jar + - io.netty-netty-resolver-dns-native-macos-4.1.134.Final-osx-aarch_64.jar + - io.netty-netty-resolver-dns-native-macos-4.1.134.Final-osx-x86_64.jar + - io.netty-netty-transport-4.1.134.Final.jar + - io.netty-netty-transport-classes-epoll-4.1.134.Final.jar + - io.netty-netty-transport-native-epoll-4.1.134.Final-linux-aarch_64.jar + - io.netty-netty-transport-native-epoll-4.1.134.Final-linux-x86_64.jar + - io.netty-netty-transport-native-unix-common-4.1.134.Final.jar - io.netty-netty-tcnative-boringssl-static-2.0.77.Final.jar - io.netty-netty-tcnative-boringssl-static-2.0.77.Final-linux-aarch_64.jar - io.netty-netty-tcnative-boringssl-static-2.0.77.Final-linux-x86_64.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 534cd205b9e7d..58c6e330f20b5 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -345,22 +345,22 @@ The Apache Software License, Version 2.0 - commons-text-1.14.0.jar - commons-compress-1.28.0.jar * Netty - - netty-buffer-4.1.133.Final.jar - - netty-codec-4.1.133.Final.jar - - netty-codec-dns-4.1.133.Final.jar - - netty-codec-http-4.1.133.Final.jar - - netty-codec-socks-4.1.133.Final.jar - - netty-codec-haproxy-4.1.133.Final.jar - - netty-common-4.1.133.Final.jar - - netty-handler-4.1.133.Final.jar - - netty-handler-proxy-4.1.133.Final.jar - - netty-resolver-4.1.133.Final.jar - - netty-resolver-dns-4.1.133.Final.jar - - netty-transport-4.1.133.Final.jar - - netty-transport-classes-epoll-4.1.133.Final.jar - - netty-transport-native-epoll-4.1.133.Final-linux-aarch_64.jar - - netty-transport-native-epoll-4.1.133.Final-linux-x86_64.jar - - netty-transport-native-unix-common-4.1.133.Final.jar + - netty-buffer-4.1.134.Final.jar + - netty-codec-4.1.134.Final.jar + - netty-codec-dns-4.1.134.Final.jar + - netty-codec-http-4.1.134.Final.jar + - netty-codec-socks-4.1.134.Final.jar + - netty-codec-haproxy-4.1.134.Final.jar + - netty-common-4.1.134.Final.jar + - netty-handler-4.1.134.Final.jar + - netty-handler-proxy-4.1.134.Final.jar + - netty-resolver-4.1.134.Final.jar + - netty-resolver-dns-4.1.134.Final.jar + - netty-transport-4.1.134.Final.jar + - netty-transport-classes-epoll-4.1.134.Final.jar + - netty-transport-native-epoll-4.1.134.Final-linux-aarch_64.jar + - netty-transport-native-epoll-4.1.134.Final-linux-x86_64.jar + - netty-transport-native-unix-common-4.1.134.Final.jar - netty-tcnative-boringssl-static-2.0.77.Final.jar - netty-tcnative-boringssl-static-2.0.77.Final-linux-aarch_64.jar - netty-tcnative-boringssl-static-2.0.77.Final-linux-x86_64.jar @@ -371,9 +371,9 @@ The Apache Software License, Version 2.0 - netty-incubator-transport-classes-io_uring-0.0.26.Final.jar - netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar - netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar - - netty-resolver-dns-classes-macos-4.1.133.Final.jar - - netty-resolver-dns-native-macos-4.1.133.Final-osx-aarch_64.jar - - netty-resolver-dns-native-macos-4.1.133.Final-osx-x86_64.jar + - netty-resolver-dns-classes-macos-4.1.134.Final.jar + - netty-resolver-dns-native-macos-4.1.134.Final-osx-aarch_64.jar + - netty-resolver-dns-native-macos-4.1.134.Final-osx-x86_64.jar * Prometheus client - simpleclient-0.16.0.jar - simpleclient_log4j2-0.16.0.jar diff --git a/pom.xml b/pom.xml index 8aca0a9da0a9f..3766bdcc06dc4 100644 --- a/pom.xml +++ b/pom.xml @@ -187,7 +187,7 @@ flexible messaging model and an intuitive client API. 1.1.10.8 4.1.12.1 5.7.1 - 4.1.133.Final + 4.1.134.Final 0.0.26.Final 12.1.9 From b7b57e0e6c499140be304f0fd775ff6ffa5b6012 Mon Sep 17 00:00:00 2001 From: Dream95 <864197662@qq.com> Date: Mon, 25 May 2026 20:28:29 +0800 Subject: [PATCH 006/213] [fix][fn] Fix Go function runtime to continue after user exceptions and add neg-ack tests (#25867) Signed-off-by: Dream95 (cherry picked from commit a5c1029668e182b05579d223683d5e6ea82f84fe) --- pulsar-function-go/pf/instance.go | 41 +++++++++++++------ pulsar-function-go/pf/instance_test.go | 32 +++++++++++++++ .../latest-version-image/Dockerfile | 1 + .../exceptionFunc/exceptionFunc.go | 41 +++++++++++++++++++ .../functions/PulsarFunctionsTest.java | 4 ++ .../functions/PulsarFunctionsTestBase.java | 1 + .../functions/go/PulsarFunctionsGoTest.java | 5 +++ 7 files changed, 112 insertions(+), 13 deletions(-) create mode 100644 tests/docker-images/latest-version-image/go-examples/exceptionFunc/exceptionFunc.go diff --git a/pulsar-function-go/pf/instance.go b/pulsar-function-go/pf/instance.go index af8a4e0157b76..2cdfc8a6e9497 100644 --- a/pulsar-function-go/pf/instance.go +++ b/pulsar-function-go/pf/instance.go @@ -164,7 +164,6 @@ CLOSE: case cm := <-channel: msgInput := cm.Message atMostOnce := gi.context.instanceConf.funcDetails.ProcessingGuarantees == pb.ProcessingGuarantees_ATMOST_ONCE - atLeastOnce := gi.context.instanceConf.funcDetails.ProcessingGuarantees == pb.ProcessingGuarantees_ATLEAST_ONCE autoAck := gi.context.instanceConf.funcDetails.AutoAck //nolint:staticcheck if autoAck && atMostOnce { gi.ackInputMessage(msgInput) @@ -177,12 +176,8 @@ CLOSE: output, err := gi.handlerMsg(msgInput) if err != nil { - log.Errorf("handler message error:%v", err) - if autoAck && atLeastOnce { - gi.nackInputMessage(msgInput) - } - gi.stats.incrTotalUserExceptions(err) - return err + gi.handleUserError(msgInput, err) + continue } gi.stats.processTimeEnd() @@ -391,6 +386,29 @@ func (gi *goInstance) setupConsumer() (chan pulsar.ConsumerMessage, error) { return channel, nil } +func (gi *goInstance) shouldNackInputOnFailure() bool { + guarantee := gi.context.instanceConf.funcDetails.ProcessingGuarantees + return guarantee == pb.ProcessingGuarantees_ATLEAST_ONCE || + guarantee == pb.ProcessingGuarantees_MANUAL +} + +func (gi *goInstance) handleUserError(msgInput pulsar.Message, err error) { + log.Errorf("handler message error:%v", err) + if gi.shouldNackInputOnFailure() { + gi.nackInputMessage(msgInput) + } + gi.stats.incrTotalUserExceptions(err) + gi.stats.processTimeEnd() +} + +func (gi *goInstance) handlePublishError(msgInput pulsar.Message, err error) { + if gi.context.instanceConf.funcDetails.ProcessingGuarantees == pb.ProcessingGuarantees_ATLEAST_ONCE { + gi.nackInputMessage(msgInput) + } + gi.stats.incrTotalSysExceptions(err) + log.Errorf("failed to publish output message: %v", err) +} + func (gi *goInstance) handlerMsg(input pulsar.Message) (output []byte, err error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -420,11 +438,8 @@ func (gi *goInstance) processResult(msgInput pulsar.Message, output []byte) { // semantics, ensure we nack so someone else can get it, in case we are the only handler. Then mark // exception and fail out. if err != nil { - if autoAck && atLeastOnce { - gi.nackInputMessage(msgInput) - } - gi.stats.incrTotalSysExceptions(err) - log.Fatal(err) + gi.handlePublishError(msgInput, err) + return } // Otherwise the message succeeded. If the SDK is entrusted with responding and we are using // atLeastOnce delivery semantics, ack the message. @@ -437,7 +452,7 @@ func (gi *goInstance) processResult(msgInput pulsar.Message, output []byte) { return } - // No output from the function or no output topic. Ack if we need to and mark the success before rturning. + // No output from the function or no output topic. Ack if we need to and mark the success before returning. if autoAck && atLeastOnce { gi.ackInputMessage(msgInput) } diff --git a/pulsar-function-go/pf/instance_test.go b/pulsar-function-go/pf/instance_test.go index bf45ae3a8917e..447d3a22f8d88 100644 --- a/pulsar-function-go/pf/instance_test.go +++ b/pulsar-function-go/pf/instance_test.go @@ -27,6 +27,8 @@ import ( "time" "github.com/stretchr/testify/assert" + + pb "github.com/apache/pulsar/pulsar-function-go/pb" ) func testProcessSpawnerHealthCheckTimer( @@ -115,3 +117,33 @@ func Test_goInstance_handlerMsg(t *testing.T) { assert.Equal(t, "output", string(output)) assert.Equal(t, message, fc.record) } + +func newTestGoInstance(guarantee pb.ProcessingGuarantees) *goInstance { + return &goInstance{ + context: &FunctionContext{ + instanceConf: &instanceConf{ + funcDetails: pb.FunctionDetails{ + ProcessingGuarantees: guarantee, + }, + }, + }, + } +} + +func TestShouldNackInputOnFailure(t *testing.T) { + tests := []struct { + name string + guarantee pb.ProcessingGuarantees + want bool + }{ + {"atLeastOnce", pb.ProcessingGuarantees_ATLEAST_ONCE, true}, + {"manual", pb.ProcessingGuarantees_MANUAL, true}, + {"atMostOnce", pb.ProcessingGuarantees_ATMOST_ONCE, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + instance := newTestGoInstance(tt.guarantee) + assert.Equal(t, tt.want, instance.shouldNackInputOnFailure()) + }) + } +} diff --git a/tests/docker-images/latest-version-image/Dockerfile b/tests/docker-images/latest-version-image/Dockerfile index 07d6a29bb8728..154708bf783b2 100644 --- a/tests/docker-images/latest-version-image/Dockerfile +++ b/tests/docker-images/latest-version-image/Dockerfile @@ -24,6 +24,7 @@ ARG GOLANG_IMAGE FROM $GOLANG_IMAGE as pulsar-function-go COPY target/pulsar-function-go/ /go/src/github.com/apache/pulsar/pulsar-function-go +COPY go-examples/exceptionFunc/ /go/src/github.com/apache/pulsar/pulsar-function-go/examples/exceptionFunc/ RUN cd /go/src/github.com/apache/pulsar/pulsar-function-go && go install ./... RUN cd /go/src/github.com/apache/pulsar/pulsar-function-go/pf && go install RUN cd /go/src/github.com/apache/pulsar/pulsar-function-go/examples && go install ./... diff --git a/tests/docker-images/latest-version-image/go-examples/exceptionFunc/exceptionFunc.go b/tests/docker-images/latest-version-image/go-examples/exceptionFunc/exceptionFunc.go new file mode 100644 index 0000000000000..80ace6e21b9b4 --- /dev/null +++ b/tests/docker-images/latest-version-image/go-examples/exceptionFunc/exceptionFunc.go @@ -0,0 +1,41 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package main + +import ( + "context" + "errors" + + "github.com/apache/pulsar/pulsar-function-go/pf" +) + +var i int + +func HandleException(ctx context.Context, in []byte) ([]byte, error) { + i++ + if i%10 == 0 { + return nil, errors.New("test") + } + return []byte(string(in) + "!"), nil +} + +func main() { + pf.Start(HandleException) +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index a46f312518a83..dde4f274b2a52 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -400,6 +400,10 @@ protected void testFunctionNegAck(Runtime runtime) throws Exception { submitFunction( runtime, inputTopicName, outputTopicName, functionName, EXCEPTION_FUNCTION_PYTHON_FILE, EXCEPTION_PYTHON_CLASS, schema, null); + } else if (runtime == Runtime.GO) { + submitFunction( + runtime, inputTopicName, outputTopicName, functionName, EXCEPTION_GO_FILE, + null, schema, null); } else { submitFunction( runtime, inputTopicName, outputTopicName, functionName, null, EXCEPTION_JAVA_CLASS, schema, null); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java index 288ced63ae5eb..84b31dca9a021 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java @@ -78,6 +78,7 @@ public abstract class PulsarFunctionsTestBase extends PulsarTestSuite { public static final String EXCLAMATION_GO_FILE = "exclamationFunc"; public static final String PUBLISH_FUNCTION_GO_FILE = "exclamationFunc"; + public static final String EXCEPTION_GO_FILE = "exceptionFunc"; public static final String LOGGING_JAVA_CLASS = "org.apache.pulsar.functions.api.examples.LoggingFunction"; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoTest.java index 0550fd94ebee2..6e631adafbf7f 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoTest.java @@ -39,4 +39,9 @@ public void testGoExclamationMultiInputsFunction() throws Exception { testExclamationFunction(Runtime.GO, false, false, true, false); } + @Test(groups = {"go_function", "function"}) + public void testGoFunctionNegAck() throws Exception { + testFunctionNegAck(Runtime.GO); + } + } From cd7feee5c760b48828322e9c870a4c6cd3b427b2 Mon Sep 17 00:00:00 2001 From: Oneby Wang <44369297+oneby-wang@users.noreply.github.com> Date: Mon, 25 May 2026 20:28:47 +0800 Subject: [PATCH 007/213] [fix][test] Fix flaky ProducerCleanupTest timer cleanup (#25864) (cherry picked from commit 2e02b7830ee14e8d3476dba03f7fa6fd0da175b8) --- .../org/apache/pulsar/client/api/ProducerCleanupTest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerCleanupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerCleanupTest.java index 5ad3c85441b6a..5508de7f4cf51 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerCleanupTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerCleanupTest.java @@ -22,6 +22,7 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -50,8 +51,9 @@ public void testAllTimerTaskShouldCanceledAfterProducerClosed() throws PulsarCli .sendTimeout(1, TimeUnit.SECONDS) .create(); producer.close(); - Thread.sleep(2000); HashedWheelTimer timer = (HashedWheelTimer) ((PulsarClientImpl) pulsarClient).timer(); - Assert.assertEquals(timer.pendingTimeouts(), 0); + Awaitility.await() + .atMost(10, TimeUnit.SECONDS) + .untilAsserted(() -> Assert.assertEquals(timer.pendingTimeouts(), 0)); } } From 58e03d6cf3766f3d073b6952763194924c572761 Mon Sep 17 00:00:00 2001 From: Oneby Wang <44369297+oneby-wang@users.noreply.github.com> Date: Sat, 30 May 2026 16:16:16 +0800 Subject: [PATCH 008/213] [fix][test] Fix flaky PulsarFunctionTlsTest.testFunctionsCreation() test (#25889) (cherry picked from commit a6af80198f26d31e991e880db3f9b7d601a579c2) --- .../worker/PulsarFunctionTlsTest.java | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java index 50d81c6efbf82..f2e49c4d7b164 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java @@ -20,8 +20,6 @@ import static org.apache.pulsar.common.util.PortManager.nextLockedFreePort; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertTrue; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; @@ -42,10 +40,10 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationProviderTls; import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.impl.auth.AuthenticationTls; import org.apache.pulsar.common.functions.FunctionConfig; -import org.apache.pulsar.common.functions.WorkerInfo; import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.util.ClassLoaderUtils; import org.apache.pulsar.common.util.ObjectMapperFactory; @@ -260,19 +258,16 @@ public void testFunctionsCreation() throws Exception { log.info(" -------- Start test function : {}", functionName); - int finalI = i; - Awaitility.await().atMost(1, TimeUnit.MINUTES).pollInterval(1, TimeUnit.SECONDS).untilAsserted(() -> { - final PulsarWorkerService workerService = ((PulsarWorkerService) fnWorkerServices[finalI]); - final LeaderService leaderService = workerService.getLeaderService(); - assertNotNull(leaderService); - if (leaderService.isLeader()) { - assertTrue(true); - } else { - final WorkerInfo workerInfo = workerService.getMembershipManager().getLeader(); - assertTrue(workerInfo != null - && !workerInfo.getWorkerId().equals(workerService.getWorkerConfig().getWorkerId())); - } - }); + final PulsarAdmin createAdmin = pulsarAdmins[i]; + // During function-worker leadership election/switchover, the coordination topic can already point to + // the new leader while that worker is still finishing its leader initialization. In that short window + // the internal /functions/leader request returns a transient 503 "Leader not yet ready", so retry only + // that condition and let all other failures surface immediately. + Awaitility.await().atMost(1, TimeUnit.MINUTES) + .pollInterval(1, TimeUnit.SECONDS) + .ignoreExceptionsMatching(PulsarFunctionTlsTest::isLeaderNotReady) + .untilAsserted(() -> createAdmin.functions() + .createFunctionWithUrl(functionConfig, jarFilePathUrl)); pulsarAdmins[i].functions().createFunctionWithUrl( functionConfig, jarFilePathUrl ); @@ -292,6 +287,14 @@ public void testFunctionsCreation() throws Exception { } } + private static boolean isLeaderNotReady(Throwable e) { + return e instanceof PulsarAdminException + && ((PulsarAdminException) e).getStatusCode() == 503 + && String.valueOf(((PulsarAdminException) e).getHttpError()) + .contains("Leader not yet ready"); + } + + protected static FunctionConfig createFunctionConfig( String jarFile, String tenant, From 5e538368fb0468c4c6d389d3887429003b270238 Mon Sep 17 00:00:00 2001 From: Oneby Wang <44369297+oneby-wang@users.noreply.github.com> Date: Mon, 25 May 2026 20:29:10 +0800 Subject: [PATCH 009/213] [fix][broker] Fix compaction cursor reset may lose mark-delete properties (#25862) (cherry picked from commit 9b15504a696e3ea21426b9e7b92a1dd613120da5) --- .../mledger/impl/ManagedCursorImpl.java | 2 +- .../mledger/impl/ManagedCursorTest.java | 84 +++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 790c81cadc9ba..6da7b5308b368 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -1669,7 +1669,7 @@ public void operationFailed(ManagedLedgerException exception) { persistentMarkDeletePosition = null; inProgressMarkDeletePersistPosition = null; - internalAsyncMarkDelete(newMarkDeletePosition, isCompactionCursor() ? getProperties() : Collections.emptyMap(), + internalAsyncMarkDelete(newMarkDeletePosition, isCompactionCursor() ? null : Collections.emptyMap(), new MarkDeleteCallback() { @Override public void markDeleteComplete(Object ctx) { diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java index 72bd81c08b025..0865d7ba1677f 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java @@ -23,6 +23,7 @@ import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.eq; @@ -6017,6 +6018,89 @@ public void deleteFailed(ManagedLedgerException exception, Object ctx) { assertEquals(properties.get(propertyKey), lastIndex - 1); } + @Test + @SuppressWarnings("unchecked") + public void testCompactionCursorResetNeverLoseMarkDeleteProperties() throws Exception { + @Cleanup + ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open( + "testCompactionCursorResetNeverLoseMarkDeleteProperties", + new ManagedLedgerConfig().setMaxEntriesPerLedger(10)); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("__compaction"); + ManagedCursorImpl spyCursor = spy(cursor); + ledger.getCursors().removeCursor(cursor.getName()); + ledger.getCursors().add(spyCursor, null); + + ledger.addEntry("entry-1".getBytes(Encoding)); + Position markDeletePosition = ledger.addEntry("entry-2".getBytes(Encoding)); + + String compactedLedgerProperty = "CompactedTopicLedger"; + Map properties = Map.of(compactedLedgerProperty, 123456L); + + CountDownLatch markDeleteEntered = new CountDownLatch(1); + CountDownLatch resetEntered = new CountDownLatch(1); + CountDownLatch markDeleteReturned = new CountDownLatch(1); + CountDownLatch markDeleteCompleted = new CountDownLatch(1); + CountDownLatch resetCompleted = new CountDownLatch(1); + + doAnswer(invocation -> { + Map invocationProperties = invocation.getArgument(1); + if (invocationProperties != null && invocationProperties.containsKey(compactedLedgerProperty)) { + // Hold the compaction mark-delete after it enters internalAsyncMarkDelete, but before its + // properties can update lastMarkDeleteEntry. + markDeleteEntered.countDown(); + assertTrue(resetEntered.await(5, TimeUnit.SECONDS)); + try { + return invocation.callRealMethod(); + } finally { + markDeleteReturned.countDown(); + } + } + + if (invocationProperties == null || invocationProperties.isEmpty()) { + // Let reset capture its properties argument first, then persist it only after the compaction + // mark-delete has completed the real internalAsyncMarkDelete call. + resetEntered.countDown(); + assertTrue(markDeleteReturned.await(5, TimeUnit.SECONDS)); + return invocation.callRealMethod(); + } + + return invocation.callRealMethod(); + }).when(spyCursor).internalAsyncMarkDelete(any(Position.class), nullable(Map.class), + any(MarkDeleteCallback.class), nullable(Object.class), nullable(Runnable.class)); + + // Start compaction mark-delete from another thread because the spy intentionally blocks it. + CompletableFuture.runAsync(() -> spyCursor.asyncMarkDelete( + markDeletePosition, properties, new MarkDeleteCallback() { + @Override + public void markDeleteComplete(Object ctx) { + markDeleteCompleted.countDown(); + } + + @Override + public void markDeleteFailed(ManagedLedgerException exception, Object ctx) { + } + }, null)); + + assertTrue(markDeleteEntered.await(5, TimeUnit.SECONDS)); + // Reset the compaction cursor while the previous mark-delete with properties is still in progress. + spyCursor.asyncResetCursor(markDeletePosition, false, new AsyncCallbacks.ResetCursorCallback() { + @Override + public void resetComplete(Object ctx) { + resetCompleted.countDown(); + } + + @Override + public void resetFailed(ManagedLedgerException exception, Object ctx) { + } + }); + + assertTrue(markDeleteCompleted.await(5, TimeUnit.SECONDS)); + assertTrue(resetCompleted.await(5, TimeUnit.SECONDS)); + + assertEquals(spyCursor.getMarkDeletedPosition(), markDeletePosition); + assertEquals(spyCursor.getProperties(), properties); + } + class TestPulsarMockBookKeeper extends PulsarMockBookKeeper { Map ledgerErrors = new HashMap<>(); From e533a3caf7c43ae78ec3d6d372b02ebec7a7161d Mon Sep 17 00:00:00 2001 From: Pratik Katti <90851204+pratt4@users.noreply.github.com> Date: Mon, 25 May 2026 20:58:25 +0530 Subject: [PATCH 010/213] Return 400 for invalid reader messageId query parameter (#25865) (cherry picked from commit b406518fd2633c842d63d31ea16fe94e015ccd88) --- .../pulsar/websocket/ReaderHandler.java | 36 +++++-- .../pulsar/websocket/ReaderHandlerTest.java | 97 ++++++++++++++++++- 2 files changed, 121 insertions(+), 12 deletions(-) diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java index 7cee6005f05d6..c4bb952c62886 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ReaderHandler.java @@ -121,11 +121,17 @@ public ReaderHandler(WebSocketService service, HttpServletRequest request, Jetty } allowConnect = true; } catch (Exception e) { - log.warn("[{}:{}] Failed in creating reader {} on topic {}", request.getRemoteAddr(), - request.getRemotePort(), subscription, topic, e); + int errorCode = getErrorCode(e); + boolean isKnownError = errorCode != HttpServletResponse.SC_INTERNAL_SERVER_ERROR; + if (isKnownError) { + log.warn("[{}:{}] Failed in creating reader {} on topic {}: {}", request.getRemoteAddr(), + request.getRemotePort(), subscription, topic, e.getMessage()); + } else { + log.error("[{}:{}] Failed in creating reader {} on topic {}", request.getRemoteAddr(), + request.getRemotePort(), subscription, topic, e); + } try { - response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, - "Failed to create reader: " + e.getMessage()); + response.sendError(errorCode, getErrorMessage(e)); } catch (IOException e1) { log.warn("[{}:{}] Failed to send error: {}", request.getRemoteAddr(), request.getRemotePort(), e1.getMessage(), e1); @@ -340,13 +346,25 @@ private int getReceiverQueueSize() { return size; } - private MessageId getMessageId() throws IOException { + private MessageId getMessageId() { MessageId messageId = MessageId.latest; - if (isNotBlank(queryParams.get("messageId"))) { - if (queryParams.get("messageId").equals("earliest")) { + String messageIdParam = queryParams.get("messageId"); + if (isNotBlank(messageIdParam)) { + if (messageIdParam.equals("earliest")) { messageId = MessageId.earliest; - } else if (!queryParams.get("messageId").equals("latest")) { - messageId = MessageIdImpl.fromByteArray(Base64.getDecoder().decode(queryParams.get("messageId"))); + } else if (!messageIdParam.equals("latest")) { + final byte[] decoded; + try { + decoded = Base64.getDecoder().decode(messageIdParam); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid messageId base64 value", e); + } + + try { + messageId = MessageIdImpl.fromByteArray(decoded); + } catch (IOException | RuntimeException e) { + throw new IllegalArgumentException("Invalid messageId value", e); + } } } return messageId; diff --git a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/ReaderHandlerTest.java b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/ReaderHandlerTest.java index a79899ab6fac7..9db1561ec5a9a 100644 --- a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/ReaderHandlerTest.java +++ b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/ReaderHandlerTest.java @@ -21,17 +21,22 @@ import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyInt; import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.IOException; +import java.util.Base64; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.function.Function; import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.PulsarClient; @@ -43,12 +48,98 @@ import org.apache.pulsar.client.impl.MultiTopicsConsumerImpl; import org.apache.pulsar.client.impl.MultiTopicsReaderImpl; import org.apache.pulsar.client.impl.ReaderImpl; +import org.apache.pulsar.common.api.proto.MessageIdData; import org.eclipse.jetty.ee8.websocket.server.JettyServerUpgradeResponse; import org.testng.Assert; import org.testng.annotations.Test; public class ReaderHandlerTest { + @Test + @SuppressWarnings("unchecked") + public void testInvalidMessageIdBase64ReturnsBadRequest() throws IOException { + WebSocketService wss = mock(WebSocketService.class); + PulsarClient mockedClient = mock(PulsarClient.class); + when(wss.getPulsarClient()).thenReturn(mockedClient); + ReaderBuilder mockedReaderBuilder = mock(ReaderBuilder.class); + when(mockedClient.newReader()).thenReturn(mockedReaderBuilder); + when(mockedReaderBuilder.topic(any())).thenReturn(mockedReaderBuilder); + // Ensure the chain doesn't NPE after startMessageId() if parsing unexpectedly succeeds. + when(mockedReaderBuilder.startMessageId(any())).thenReturn(mockedReaderBuilder); + + Map params = new HashMap<>(); + params.put("messageId", new String[] { "invalidMessageId" }); + + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getRequestURI()).thenReturn("/ws/v2/reader/persistent/my-property/my-ns/my-topic"); + when(request.getParameterMap()).thenReturn(params); + + JettyServerUpgradeResponse servletUpgradeResponse = mock(JettyServerUpgradeResponse.class); + new ReaderHandler(wss, request, servletUpgradeResponse); + + verify(servletUpgradeResponse, times(1)) + .sendError(eq(HttpServletResponse.SC_BAD_REQUEST), anyString()); + } + + @Test + @SuppressWarnings("unchecked") + public void testInvalidMessageIdBytesReturnsBadRequest() throws IOException { + WebSocketService wss = mock(WebSocketService.class); + PulsarClient mockedClient = mock(PulsarClient.class); + when(wss.getPulsarClient()).thenReturn(mockedClient); + ReaderBuilder mockedReaderBuilder = mock(ReaderBuilder.class); + when(mockedClient.newReader()).thenReturn(mockedReaderBuilder); + when(mockedReaderBuilder.topic(any())).thenReturn(mockedReaderBuilder); + // Ensure the chain doesn't NPE after startMessageId() if parsing unexpectedly succeeds. + when(mockedReaderBuilder.startMessageId(any())).thenReturn(mockedReaderBuilder); + + // "AQID" is valid Base64, but it doesn't decode into a valid Pulsar MessageId structure. + Map params = new HashMap<>(); + params.put("messageId", new String[] { "AQID" }); + + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getRequestURI()).thenReturn("/ws/v2/reader/persistent/my-property/my-ns/my-topic"); + when(request.getParameterMap()).thenReturn(params); + + JettyServerUpgradeResponse servletUpgradeResponse = mock(JettyServerUpgradeResponse.class); + new ReaderHandler(wss, request, servletUpgradeResponse); + + verify(servletUpgradeResponse, times(1)) + .sendError(eq(HttpServletResponse.SC_BAD_REQUEST), anyString()); + } + + @Test + @SuppressWarnings("unchecked") + public void testInvalidMessageIdRuntimeParseFailureReturnsBadRequest() throws IOException { + WebSocketService wss = mock(WebSocketService.class); + PulsarClient mockedClient = mock(PulsarClient.class); + when(wss.getPulsarClient()).thenReturn(mockedClient); + ReaderBuilder mockedReaderBuilder = mock(ReaderBuilder.class); + when(mockedClient.newReader()).thenReturn(mockedReaderBuilder); + when(mockedReaderBuilder.topic(any())).thenReturn(mockedReaderBuilder); + // Ensure the chain doesn't NPE after startMessageId() if parsing unexpectedly succeeds. + when(mockedReaderBuilder.startMessageId(any())).thenReturn(mockedReaderBuilder); + + MessageIdData invalidBatchMessageId = new MessageIdData() + .setLedgerId(1) + .setEntryId(2) + .setBatchIndex(0) + .setBatchSize(-1); + Map params = new HashMap<>(); + params.put("messageId", new String[] { + Base64.getEncoder().encodeToString(invalidBatchMessageId.toByteArray()) }); + + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getRequestURI()).thenReturn("/ws/v2/reader/persistent/my-property/my-ns/my-topic"); + when(request.getParameterMap()).thenReturn(params); + + JettyServerUpgradeResponse servletUpgradeResponse = mock(JettyServerUpgradeResponse.class); + new ReaderHandler(wss, request, servletUpgradeResponse); + + verify(servletUpgradeResponse, times(1)) + .sendError(eq(HttpServletResponse.SC_BAD_REQUEST), anyString()); + } + @Test @SuppressWarnings("unchecked") public void testCreateReaderImp() throws IOException { @@ -68,7 +159,7 @@ public void testCreateReaderImp() throws IOException { when(consumerImp.getSubscription()).thenReturn(subName); when(mockedReader.getConsumer()).thenReturn(consumerImp); HttpServletRequest request = mock(HttpServletRequest.class); - when(request.getRequestURI()).thenReturn("/ws/v2/producer/persistent/my-property/my-ns/my-topic"); + when(request.getRequestURI()).thenReturn("/ws/v2/reader/persistent/my-property/my-ns/my-topic"); // create reader handler JettyServerUpgradeResponse servletUpgradeResponse = mock(JettyServerUpgradeResponse.class); ReaderHandler readerHandler = new ReaderHandler(wss, request, servletUpgradeResponse); @@ -97,7 +188,7 @@ public void testCreateMultipleTopicReaderImp() throws IOException { when(consumerImp.getSubscription()).thenReturn(subName); when(mockedReader.getMultiTopicsConsumer()).thenReturn(consumerImp); HttpServletRequest request = mock(HttpServletRequest.class); - when(request.getRequestURI()).thenReturn("/ws/v2/producer/persistent/my-property/my-ns/my-topic"); + when(request.getRequestURI()).thenReturn("/ws/v2/reader/persistent/my-property/my-ns/my-topic"); // create reader handler JettyServerUpgradeResponse servletUpgradeResponse = mock(JettyServerUpgradeResponse.class); ReaderHandler readerHandler = new ReaderHandler(wss, request, servletUpgradeResponse); @@ -122,7 +213,7 @@ public void testCreateIllegalReaderImp() throws IOException { IllegalReader illegalReader = new IllegalReader(); when(mockedReaderBuilder.create()).thenReturn(illegalReader); HttpServletRequest request = mock(HttpServletRequest.class); - when(request.getRequestURI()).thenReturn("/ws/v2/producer/persistent/my-property/my-ns/my-topic"); + when(request.getRequestURI()).thenReturn("/ws/v2/reader/persistent/my-property/my-ns/my-topic"); // create reader handler JettyServerUpgradeResponse servletUpgradeResponse = spy(JettyServerUpgradeResponse.class); new ReaderHandler(wss, request, servletUpgradeResponse); From 7e75bb9472dbda29b8b513b36959dc1254405b3a Mon Sep 17 00:00:00 2001 From: Oneby Wang <44369297+oneby-wang@users.noreply.github.com> Date: Mon, 25 May 2026 23:30:50 +0800 Subject: [PATCH 011/213] [fix][broker] Fix PersistentMessageExpiryMonitor findEntryComplete() method may lose mark-delete properties in race condition (#25803) (cherry picked from commit 47eec875a81d11a030a3781cbb123e273ced5e88) Signed-off-by: Zixuan Liu --- .../PersistentMessageExpiryMonitor.java | 2 +- .../service/PersistentMessageFinderTest.java | 58 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java index bc62263adc843..b327a2e05ed39 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java @@ -260,7 +260,7 @@ public void findEntryComplete(Position position, Object ctx) { } log.info("[{}][{}] Expiring all messages until position {}", topicName, subName, position); Position prevMarkDeletePos = cursor.getMarkDeletedPosition(); - cursor.asyncMarkDelete(position, cursor.getProperties(), getMarkDeleteCallback(position), + cursor.asyncMarkDelete(position, null, getMarkDeleteCallback(position), cursor.getNumberOfEntriesInBacklog(false)); if (!Objects.equals(cursor.getMarkDeletedPosition(), prevMarkDeletePos) && subscription != null) { subscription.updateLastMarkDeleteAdvancedTimestamp(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java index cafa551ce242a..942a58da5fd95 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.service; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -36,11 +37,14 @@ import io.netty.buffer.UnpooledByteBufAllocator; import java.lang.reflect.Field; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -1088,4 +1092,58 @@ public void testGetFindPositionRange_SingleClosedLedger() { assertNull(range.getRight()); assertEquals(range.getLeft(), PositionFactory.create(1, 9)); } + + @Test + @SuppressWarnings("unchecked") + void testExpireMessagesNeverLoseMarkDeleteProperties() throws Exception { + final String ledgerAndCursorName = "testExpireMessagesNeverLoseMarkDeleteProperties"; + + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setRetentionSizeInMB(10); + config.setRetentionTime(1, TimeUnit.HOURS); + ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open(ledgerAndCursorName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor(ledgerAndCursorName); + ManagedCursorImpl spyCursor = spy(cursor); + + Position pos1 = ledger.addEntry(createMessageWrittenToLedger("msg-1")); + Position pos2 = ledger.addEntry(createMessageWrittenToLedger("msg-2")); + + CountDownLatch expiryMarkDeleteEnteredLatch = new CountDownLatch(1); + CountDownLatch cursorMarkDeleteCompletedLatch = new CountDownLatch(1); + CountDownLatch expiryMarkDeleteCompletedLatch = new CountDownLatch(1); + + doAnswer(invocation -> { + Map invocationProperties = invocation.getArgument(1); + // Pause the expiry-triggered mark-delete so the user markDelete() can complete first. + if (invocationProperties == null || invocationProperties.isEmpty()) { + expiryMarkDeleteEnteredLatch.countDown(); + assertTrue(cursorMarkDeleteCompletedLatch.await(5, TimeUnit.SECONDS)); + try { + return invocation.callRealMethod(); + } finally { + expiryMarkDeleteCompletedLatch.countDown(); + } + } + + return invocation.callRealMethod(); + }).when(spyCursor) + .asyncMarkDelete(any(Position.class), nullable(Map.class), any(AsyncCallbacks.MarkDeleteCallback.class), + nullable(Object.class)); + + PersistentTopic topic = mockPersistentTopic("topicname"); + PersistentMessageExpiryMonitor monitor = new PersistentMessageExpiryMonitor(topic, + spyCursor.getName(), spyCursor, null); + + CompletableFuture.runAsync(() -> monitor.findEntryComplete(pos2, null)); + assertTrue(expiryMarkDeleteEnteredLatch.await(5, TimeUnit.SECONDS)); + + Map properties = new HashMap<>(); + properties.put("test-property", 1L); + spyCursor.markDelete(pos1, properties); + cursorMarkDeleteCompletedLatch.countDown(); + + assertTrue(expiryMarkDeleteCompletedLatch.await(5, TimeUnit.SECONDS)); + assertEquals(spyCursor.getMarkDeletedPosition(), pos2); + assertEquals(spyCursor.getProperties(), properties); + } } From ea9bbae1235e2868d76d37732e971af7d38850d1 Mon Sep 17 00:00:00 2001 From: Yike Xiao Date: Wed, 27 May 2026 23:18:58 +0800 Subject: [PATCH 012/213] [fix][bk] Fix NPE in IsolatedBookieEnsemblePlacementPolicy when policy class does not match (#25825) Co-authored-by: Claude Sonnet 4.6 (cherry picked from commit b93fe9e78f3929b52c33475aa2962a0450a7e2de) --- ...IsolatedBookieEnsemblePlacementPolicy.java | 28 ++-- ...atedBookieEnsemblePlacementPolicyTest.java | 136 ++++++++++++++++++ 2 files changed, 152 insertions(+), 12 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicy.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicy.java index 4ef1c594be444..f2c1b5e736191 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicy.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicy.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.bookie.rackawareness; +import static java.util.Collections.emptySet; import static org.apache.pulsar.bookie.rackawareness.BookieRackAffinityMapping.METADATA_STORE_INSTANCE; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Sets; @@ -164,11 +165,13 @@ private static Optional getEnsemblePlacementPolic return Optional.empty(); } - private static Pair, Set> getIsolationGroup( + @VisibleForTesting + Pair, Set> getIsolationGroup( EnsemblePlacementPolicyConfig ensemblePlacementPolicyConfig) { - MutablePair, Set> pair = new MutablePair<>(); - String className = IsolatedBookieEnsemblePlacementPolicy.class.getName(); - if (ensemblePlacementPolicyConfig.getPolicyClass().getName().equals(className)) { + // Retain compatibility with ZkIsolatedBookieEnsemblePlacementPolicy + Class policyClass = ensemblePlacementPolicyConfig.getPolicyClass(); + if (IsolatedBookieEnsemblePlacementPolicy.class.isAssignableFrom(policyClass)) { + MutablePair, Set> pair = new MutablePair<>(emptySet(), emptySet()); Map properties = ensemblePlacementPolicyConfig.getProperties(); String primaryIsolationGroupString = ConfigurationStringUtil .castToString(properties.getOrDefault(ISOLATION_BOOKIE_GROUPS, "")); @@ -176,21 +179,22 @@ private static Pair, Set> getIsolationGroup( .castToString(properties.getOrDefault(SECONDARY_ISOLATION_BOOKIE_GROUPS, "")); if (!primaryIsolationGroupString.isEmpty()) { pair.setLeft(Sets.newHashSet(primaryIsolationGroupString.split(","))); - } else { - pair.setLeft(Collections.emptySet()); } if (!secondaryIsolationGroupString.isEmpty()) { pair.setRight(Sets.newHashSet(secondaryIsolationGroupString.split(","))); - } else { - pair.setRight(Collections.emptySet()); } + return pair; + } else { + log.info("The ensemble placement policy class [{}] is not compatible with " + + "IsolatedBookieEnsemblePlacementPolicy, fallback to use defaultIsolationGroups", + ensemblePlacementPolicyConfig.getPolicyClass().getName()); + return defaultIsolationGroups; } - return pair; } @VisibleForTesting Set getExcludedBookiesWithIsolationGroups(int ensembleSize, - Pair, Set> isolationGroups) { + Pair, Set> isolationGroups) { Set excludedBookies = new HashSet<>(); if (isolationGroups != null && isolationGroups.getLeft().contains(PULSAR_SYSTEM_TOPIC_ISOLATION_GROUP)) { return excludedBookies; @@ -213,8 +217,8 @@ Set getExcludedBookiesWithIsolationGroups(int ensembleSize, return excludedBookies; } int totalAvailableBookiesInPrimaryGroup = 0; - Set primaryIsolationGroup = Collections.emptySet(); - Set secondaryIsolationGroup = Collections.emptySet(); + Set primaryIsolationGroup = emptySet(); + Set secondaryIsolationGroup = emptySet(); Set primaryGroupBookies = new HashSet<>(); if (isolationGroups != null) { primaryIsolationGroup = isolationGroups.getLeft(); diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java index 0dc996c7d7def..fc67395c813c4 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java @@ -42,12 +42,14 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.client.BKException.BKNotEnoughBookiesException; +import org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy; import org.apache.bookkeeper.conf.ClientConfiguration; import org.apache.bookkeeper.feature.SettableFeatureProvider; import org.apache.bookkeeper.net.BookieId; import org.apache.bookkeeper.net.BookieSocketAddress; import org.apache.bookkeeper.stats.NullStatsLogger; import org.apache.commons.lang3.tuple.MutablePair; +import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.common.policies.data.BookieInfo; import org.apache.pulsar.common.policies.data.BookiesRackConfiguration; import org.apache.pulsar.common.policies.data.EnsemblePlacementPolicyConfig; @@ -57,6 +59,7 @@ import org.apache.pulsar.metadata.api.MetadataStoreFactory; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.apache.pulsar.metadata.cache.impl.MetadataCacheImpl; +import org.apache.pulsar.zookeeper.ZkIsolatedBookieEnsemblePlacementPolicy; import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -844,6 +847,139 @@ public void testGetExcludedBookiesWithIsolationGroups() throws Exception { assertTrue(blacklist.isEmpty()); } + /** + * Regression test for the NPE reported in the stack trace below. When custom metadata carries an + * {@link EnsemblePlacementPolicyConfig} whose policy class does NOT match + * {@link IsolatedBookieEnsemblePlacementPolicy}, the old {@code getIsolationGroup()} returned a + * {@code MutablePair} with {@code null} left/right, which caused a {@link NullPointerException} in + * {@code getExcludedBookiesWithIsolationGroups} when {@code getLeft().contains(...)} was called. + * + * 
+     * java.lang.NullPointerException: Cannot invoke "java.util.Set.contains(Object)"
+     *     because the return value of "org.apache.commons.lang3.tuple.Pair.getLeft()" is null
+     *     at IsolatedBookieEnsemblePlacementPolicy.getExcludedBookiesWithIsolationGroups(...)
+     *     at IsolatedBookieEnsemblePlacementPolicy.getExcludedBookies(...)
+     *     at IsolatedBookieEnsemblePlacementPolicy.replaceBookie(...)
+     *
+ */ + @Test + public void testReplaceBookieWithNonMatchingPolicyClassShouldNotThrowNPE() throws Exception { + Map> bookieMapping = new HashMap<>(); + Map group1 = new HashMap<>(); + group1.put(BOOKIE1, BookieInfo.builder().rack("rack0").build()); + group1.put(BOOKIE2, BookieInfo.builder().rack("rack1").build()); + group1.put(BOOKIE3, BookieInfo.builder().rack("rack0").build()); + group1.put(BOOKIE4, BookieInfo.builder().rack("rack1").build()); + bookieMapping.put("group1", group1); + + store.put(BookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH, jsonMapper.writeValueAsBytes(bookieMapping), + Optional.empty()).join(); + + IsolatedBookieEnsemblePlacementPolicy isolationPolicy = new IsolatedBookieEnsemblePlacementPolicy(); + ClientConfiguration bkClientConf = new ClientConfiguration(); + bkClientConf.setProperty(BookieRackAffinityMapping.METADATA_STORE_INSTANCE, store); + bkClientConf.setProperty(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, "group1"); + isolationPolicy.initialize(bkClientConf, Optional.empty(), timer, SettableFeatureProvider.DISABLE_ALL, + NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); + isolationPolicy.onClusterChanged(writableBookies, readOnlyBookies); + + // Use a policy class that does NOT match IsolatedBookieEnsemblePlacementPolicy. + // In the old code this caused getIsolationGroup() to return a MutablePair with null left/right, + // triggering NPE at the getLeft().contains() call in getExcludedBookiesWithIsolationGroups. + EnsemblePlacementPolicyConfig policyConfig = new EnsemblePlacementPolicyConfig( + RackawareEnsemblePlacementPolicy.class, Collections.emptyMap()); + Map customMetadata = new HashMap<>(); + customMetadata.put(EnsemblePlacementPolicyConfig.ENSEMBLE_PLACEMENT_POLICY_CONFIG, policyConfig.encode()); + + BookieId bookie1Id = new BookieSocketAddress(BOOKIE1).toBookieId(); + BookieId bookie2Id = new BookieSocketAddress(BOOKIE2).toBookieId(); + + // Must not throw NullPointerException; BKNotEnoughBookiesException is acceptable. + isolationPolicy.replaceBookie(2, 2, 2, customMetadata, + Arrays.asList(bookie1Id, bookie2Id), bookie2Id, null); + } + + /** + * Verifies that {@link IsolatedBookieEnsemblePlacementPolicy#getIsolationGroup} treats + * {@link ZkIsolatedBookieEnsemblePlacementPolicy} (a subclass) exactly like + * {@link IsolatedBookieEnsemblePlacementPolicy} itself when reading isolation groups from + * {@link EnsemblePlacementPolicyConfig} properties. + * + *

Legacy Pulsar clusters may have persisted {@code EnsemblePlacementPolicyConfig} entries whose + * {@code policyClass} field is set to {@code ZkIsolatedBookieEnsemblePlacementPolicy}. The + * {@code isAssignableFrom} check in {@code getIsolationGroup} must recognise this subclass so that + * the isolation groups are read from the stored properties rather than falling back to the + * policy-level defaults. + */ + @Test + public void testGetIsolationGroupWithZkCompatiblePolicyClass() throws Exception { + // Group1 → default isolation group configured on the policy. + // Group2 → isolation group carried inside the custom metadata (ZkIsolated class). + final String defaultGroup = "Group1"; + final String customGroup = "Group2"; + + Map> bookieMapping = new HashMap<>(); + Map group1 = new HashMap<>(); + group1.put(BOOKIE1, BookieInfo.builder().rack("rack0").build()); + group1.put(BOOKIE2, BookieInfo.builder().rack("rack0").build()); + Map group2 = new HashMap<>(); + group2.put(BOOKIE3, BookieInfo.builder().rack("rack1").build()); + group2.put(BOOKIE4, BookieInfo.builder().rack("rack1").build()); + bookieMapping.put(defaultGroup, group1); + bookieMapping.put(customGroup, group2); + + store.put(BookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH, jsonMapper.writeValueAsBytes(bookieMapping), + Optional.empty()).join(); + + IsolatedBookieEnsemblePlacementPolicy isolationPolicy = new IsolatedBookieEnsemblePlacementPolicy(); + ClientConfiguration bkClientConf = new ClientConfiguration(); + bkClientConf.setProperty(BookieRackAffinityMapping.METADATA_STORE_INSTANCE, store); + bkClientConf.setProperty(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, defaultGroup); + isolationPolicy.initialize(bkClientConf, Optional.empty(), timer, SettableFeatureProvider.DISABLE_ALL, + NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); + isolationPolicy.onClusterChanged(writableBookies, readOnlyBookies); + + // --- unit-level: getIsolationGroup should parse properties, not fall back to defaults --- + Map props = new HashMap<>(); + props.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, customGroup); + props.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, "secondaryGroup"); + EnsemblePlacementPolicyConfig zkConfig = new EnsemblePlacementPolicyConfig( + ZkIsolatedBookieEnsemblePlacementPolicy.class, props); + + Pair, Set> groups = isolationPolicy.getIsolationGroup(zkConfig); + assertEquals(groups.getLeft(), Sets.newHashSet(customGroup), + "primary group must be read from ZkIsolated config properties"); + assertEquals(groups.getRight(), Sets.newHashSet("secondaryGroup"), + "secondary group must be read from ZkIsolated config properties"); + + // --- integration-level: newEnsemble must select bookies from the ZkIsolated config group --- + Map placementPolicyProperties = new HashMap<>(); + placementPolicyProperties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, customGroup); + placementPolicyProperties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, ""); + EnsemblePlacementPolicyConfig policyConfig = new EnsemblePlacementPolicyConfig( + ZkIsolatedBookieEnsemblePlacementPolicy.class, placementPolicyProperties); + Map customMetadata = new HashMap<>(); + customMetadata.put(EnsemblePlacementPolicyConfig.ENSEMBLE_PLACEMENT_POLICY_CONFIG, policyConfig.encode()); + + Set bookieIdGroup2 = new HashSet<>(); + bookieIdGroup2.add(new BookieSocketAddress(BOOKIE3).toBookieId()); + bookieIdGroup2.add(new BookieSocketAddress(BOOKIE4).toBookieId()); + + List ensemble = isolationPolicy + .newEnsemble(2, 2, 2, customMetadata, new HashSet<>()).getResult(); + assertTrue(bookieIdGroup2.containsAll(ensemble), + "ensemble should come from " + customGroup + " (ZkIsolated config), got " + ensemble); + + // Sanity-check: without custom metadata the default group1 bookies are chosen. + Set bookieIdGroup1 = new HashSet<>(); + bookieIdGroup1.add(new BookieSocketAddress(BOOKIE1).toBookieId()); + bookieIdGroup1.add(new BookieSocketAddress(BOOKIE2).toBookieId()); + List defaultEnsemble = isolationPolicy + .newEnsemble(2, 2, 2, Collections.emptyMap(), new HashSet<>()).getResult(); + assertTrue(bookieIdGroup1.containsAll(defaultEnsemble), + "default ensemble should come from " + defaultGroup + ", got " + defaultEnsemble); + } + // The policy gets the bookie info asynchronously before each query or update, when putting the bookie info into // the metadata store, the cache needs some time to receive the notification and update accordingly. private void updateBookieInfo(IsolatedBookieEnsemblePlacementPolicy isolationPolicy, byte[] bookieInfo) { From 5e35087ae2591ecabf9029e0ea0dd5282ce3025c Mon Sep 17 00:00:00 2001 From: grishaf Date: Fri, 29 May 2026 10:13:15 +0300 Subject: [PATCH 013/213] [fix][broker] Fix non-batched null-value messages not removed during topic compaction (#25817) (cherry picked from commit 1fa9e3532b4c8978b25f16b43855948b54e95d17) --- .../compaction/AbstractTwoPhaseCompactor.java | 20 +++-- .../compaction/EventTimeOrderCompactor.java | 14 +--- .../pulsar/compaction/CompactionTest.java | 74 +++++++++++++++++++ 3 files changed, 93 insertions(+), 15 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/AbstractTwoPhaseCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/AbstractTwoPhaseCompactor.java index f830d16fe162b..2550adac3aa19 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/AbstractTwoPhaseCompactor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/AbstractTwoPhaseCompactor.java @@ -446,14 +446,24 @@ private CompletableFuture addToCompactedLedger(LedgerHandle lh, RawMessage return bkf; } + /** + * Extract the partition key and the payload size for a non-batch message. + * + * @return a pair of (partitionKey, payloadSize), or null if the message has no partition key. + */ protected Pair extractKeyAndSize(RawMessage m, MessageMetadata msgMetadata) { - ByteBuf headersAndPayload = m.getHeadersAndPayload(); if (msgMetadata.hasPartitionKey()) { - int size = headersAndPayload.readableBytes(); - if (msgMetadata.hasUncompressedSize()) { - size = msgMetadata.getUncompressedSize(); + int payloadSize; + if (msgMetadata.hasNullValue() && msgMetadata.isNullValue()) { + payloadSize = 0; + } else if (msgMetadata.hasUncompressedSize()) { + payloadSize = msgMetadata.getUncompressedSize(); + } else { + ByteBuf headersAndPayload = m.getHeadersAndPayload().duplicate(); + Commands.skipMessageMetadata(headersAndPayload); + payloadSize = headersAndPayload.readableBytes(); } - return Pair.of(msgMetadata.getPartitionKey(), size); + return Pair.of(msgMetadata.getPartitionKey(), payloadSize); } else { return null; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/EventTimeOrderCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/EventTimeOrderCompactor.java index db129b54533a8..ad6b5f28d306f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/EventTimeOrderCompactor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/EventTimeOrderCompactor.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.compaction; -import io.netty.buffer.ByteBuf; import java.io.IOException; import java.util.List; import java.util.Map; @@ -139,17 +138,12 @@ protected boolean compactBatchMessage(String topic, Map keyAndSize = extractKeyAndSize(m, metadata); + if (keyAndSize == null) { return null; } + return new MessageCompactionData(m.getMessageId(), keyAndSize.getLeft(), + keyAndSize.getRight(), metadata.getEventTime()); } private List extractMessageCompactionDataFromBatch(RawMessage msg, MessageMetadata metadata) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java index 15866e3e6626c..c882fe460fd38 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java @@ -640,6 +640,80 @@ public void testBatchMessageWithNullValue() throws Exception { assertEquals(messages.get(2).getKey(), "key5"); } + /** + * Write raw non-batch entries directly to the managed ledger without + * uncompressedSize, as seen with some non-Java clients. Verifies that + * null-value tombstones remove keys during compaction. + */ + @Test + public void testNonBatchedMessageWithNullValue() throws Exception { + String topic = "persistent://my-tenant/my-ns/non-batched-message-with-null-value"; + + admin.topics().createNonPartitionedTopic(topic); + pulsarClient.newConsumer().topic(topic).subscriptionName("sub1") + .receiverQueueSize(1).readCompacted(true).subscribe().close(); + + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topic, false).join().get(); + ManagedLedgerImpl ml = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); + + long seqId = 0; + + // key1: value then null-value tombstone + ml.addEntry(buildNonBatchEntry("key1", "my-message-1".getBytes(), seqId++)); + ml.addEntry(buildNonBatchEntry("key1", null, seqId++)); + + // key2: value only (should survive) + ml.addEntry(buildNonBatchEntry("key2", "my-message-3".getBytes(), seqId++)); + + // key3: value then null-value tombstone + ml.addEntry(buildNonBatchEntry("key3", "my-message-4".getBytes(), seqId++)); + ml.addEntry(buildNonBatchEntry("key3", null, seqId++)); + + // key4: value only (should survive) + ml.addEntry(buildNonBatchEntry("key4", "my-message-6".getBytes(), seqId++)); + + compact(topic); + + List> messages = new ArrayList<>(); + try (Consumer consumer = pulsarClient.newConsumer().topic(topic) + .subscriptionName("sub1").receiverQueueSize(1).readCompacted(true).subscribe()) { + while (true) { + Message message = consumer.receive(5, TimeUnit.SECONDS); + if (message == null) { + break; + } + messages.add(message); + } + } + + assertEquals(messages.size(), 2); + assertEquals(messages.get(0).getKey(), "key2"); + assertEquals(messages.get(1).getKey(), "key4"); + } + + private byte[] buildNonBatchEntry(String key, byte[] payload, long sequenceId) { + org.apache.pulsar.common.api.proto.MessageMetadata metadata = + new org.apache.pulsar.common.api.proto.MessageMetadata(); + metadata.setPartitionKey(key); + metadata.setPublishTime(System.currentTimeMillis()); + metadata.setProducerName("test-non-batch"); + metadata.setSequenceId(sequenceId); + if (payload == null) { + metadata.setNullValue(true); + } + ByteBuf payloadBuf = io.netty.buffer.Unpooled.wrappedBuffer( + payload != null ? payload : new byte[0]); + ByteBuf entry = org.apache.pulsar.common.protocol.Commands.serializeMetadataAndPayload( + org.apache.pulsar.common.protocol.Commands.ChecksumType.Crc32c, + metadata, payloadBuf); + byte[] bytes = new byte[entry.readableBytes()]; + entry.readBytes(bytes); + entry.release(); + payloadBuf.release(); + return bytes; + } + @Test public void testWholeBatchCompactedOut() throws Exception { String topic = "persistent://my-tenant/my-ns/whole-batch-compacted-out"; From adaffc919ebf83cd80b4c4131bf62ea4e7e0712a Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Fri, 24 Apr 2026 10:53:23 +0800 Subject: [PATCH 014/213] [improve][client] Best-effort retry for individual/batch-index acks on send failure when ackReceiptEnabled=false (#25525) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> (cherry picked from commit fae3df958f74f2afb0ba398b69d94fd3e6358c4f) --- .../apache/pulsar/client/impl/ClientCnx.java | 26 +++-- ...sistentAcknowledgmentsGroupingTracker.java | 75 +++++++++++++-- .../AcknowledgementsGroupingTrackerTest.java | 94 ++++++++++++++++++- .../ClientCnxRequestTimeoutQueueTest.java | 1 + .../client/impl/ClientTestFixtures.java | 5 + 5 files changed, 174 insertions(+), 27 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index 35c49fb46a898..3982226390699 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -1053,27 +1053,23 @@ CompletableFuture sendRequestWithId(ByteBuf cmd, long requestI } private void sendRequestAndHandleTimeout(ByteBuf requestMessage, long requestId, - RequestType requestType, boolean flush, - TimedCompletableFuture future) { + RequestType requestType, boolean flush, + TimedCompletableFuture future) { pendingRequests.put(requestId, future); - if (flush) { - ctx.writeAndFlush(requestMessage).addListener(writeFuture -> { - if (!writeFuture.isSuccess()) { - if (pendingRequests.remove(requestId, future) && !future.isDone()) { - log.warn("{} Failed to send {} to broker: {}", ctx.channel(), - requestType.getDescription(), writeFuture.cause().getMessage()); - future.completeExceptionally(writeFuture.cause()); - } + (flush ? ctx.writeAndFlush(requestMessage) : ctx.write(requestMessage)).addListener(writeFuture -> { + if (!writeFuture.isSuccess()) { + if (pendingRequests.remove(requestId, future) && !future.isDone()) { + log.warn("{} Failed to send {} to broker: {}", ctx.channel(), + requestType.getDescription(), writeFuture.cause().getMessage()); + future.completeExceptionally(writeFuture.cause()); } - }); - } else { - ctx.write(requestMessage, ctx().voidPromise()); - } + } + }); requestTimeoutQueue.add(new RequestTime(requestId, requestType)); } private CompletableFuture sendRequestAndHandleTimeout(ByteBuf requestMessage, long requestId, - RequestType requestType, boolean flush) { + RequestType requestType, boolean flush) { TimedCompletableFuture future = new TimedCompletableFuture<>(); sendRequestAndHandleTimeout(requestMessage, requestId, requestType, flush, future); return future; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java index 0598dc4fb3626..0a366a759d3f4 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java @@ -26,6 +26,7 @@ import java.util.ArrayList; import java.util.BitSet; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -279,6 +280,11 @@ private CompletableFuture doIndividualAckAsync(MessageIdAdv messageId) { return CompletableFuture.completedFuture(null); } + @VisibleForTesting + int getPendingIndividualAcksSize() { + return pendingIndividualAcks.size(); + } + private CompletableFuture doIndividualBatchAck(MessageIdAdv batchMessageId, Map properties) { if (acknowledgementGroupTimeMicros == 0 || (properties != null && !properties.isEmpty())) { @@ -405,7 +411,7 @@ private CompletableFuture doImmediateBatchIndexAck(MessageIdAdv msgId, int } CompletableFuture completableFuture = newMessageAckCommandAndWrite(cnx, consumer.consumerId, - msgId.getLedgerId(), msgId.getEntryId(), bitSet, ackType, properties, true, null, null); + msgId.getLedgerId(), msgId.getEntryId(), bitSet, ackType, properties, true, null, null, null); bitSet.recycle(); return completableFuture; } @@ -441,13 +447,15 @@ private void flushAsync(ClientCnx cnx) { newMessageAckCommandAndWrite(cnx, consumer.consumerId, messageId.getLedgerId(), messageId.getEntryId(), lastCumulativeAckToFlush.getBitSetRecyclable(), AckType.Cumulative, Collections.emptyMap(), false, - (TimedCompletableFuture) this.currentCumulativeAckFuture, null); + (TimedCompletableFuture) this.currentCumulativeAckFuture, null, null); this.consumer.unAckedChunkedMessageIdSequenceMap.remove(messageId); } // Flush all individual acks List> entriesToAck = new ArrayList<>(pendingIndividualAcks.size() + pendingIndividualBatchIndexAcks.size()); + List individualAcksToFlush = new ArrayList<>(pendingIndividualAcks.size()); + Map chunkedMessageIdsToRestore = new HashMap<>(); if (!pendingIndividualAcks.isEmpty()) { if (Commands.peerSupportsMultiMessageAcknowledgment(cnx.getRemoteEndpointProtocolVersion())) { // We can send 1 single protobuf command with all individual acks @@ -456,11 +464,13 @@ private void flushAsync(ClientCnx cnx) { if (msgId == null) { break; } + individualAcksToFlush.add(msgId); // if messageId is checked then all the chunked related to that msg also processed so, ack all of // them MessageIdImpl[] chunkMsgIds = this.consumer.unAckedChunkedMessageIdSequenceMap.get(msgId); if (chunkMsgIds != null && chunkMsgIds.length > 1) { + chunkedMessageIdsToRestore.put(msgId, chunkMsgIds); for (MessageIdImpl cMsgId : chunkMsgIds) { if (cMsgId != null) { entriesToAck.add(Triple.of(cMsgId.getLedgerId(), cMsgId.getEntryId(), null)); @@ -479,14 +489,16 @@ private void flushAsync(ClientCnx cnx) { if (msgId == null) { break; } + individualAcksToFlush.add(msgId); newMessageAckCommandAndWrite(cnx, consumer.consumerId, msgId.getLedgerId(), msgId.getEntryId(), null, AckType.Individual, Collections.emptyMap(), false, - null, null); + null, null, () -> restoreIndividualAck(msgId, null)); shouldFlush = true; } } } + List> batchIndexAcksToFlush = new ArrayList<>(); while (true) { Map.Entry entry = pendingIndividualBatchIndexAcks.pollFirstEntry(); @@ -494,6 +506,7 @@ private void flushAsync(ClientCnx cnx) { // The entry has been removed in a different thread break; } + batchIndexAcksToFlush.add(entry); entriesToAck.add(Triple.of( entry.getKey().getLedgerId(), entry.getKey().getEntryId(), entry.getValue())); } @@ -502,7 +515,9 @@ private void flushAsync(ClientCnx cnx) { newMessageAckCommandAndWrite(cnx, consumer.consumerId, 0L, 0L, null, AckType.Individual, null, true, - (TimedCompletableFuture) currentIndividualAckFuture, entriesToAck); + (TimedCompletableFuture) currentIndividualAckFuture, entriesToAck, + () -> restoreIndividualAndBatchIndexAcks(individualAcksToFlush, chunkedMessageIdsToRestore, + batchIndexAcksToFlush)); shouldFlush = true; } @@ -547,19 +562,19 @@ private CompletableFuture newImmediateAckAndFlush(long consumerId, Message } } completableFuture = newMessageAckCommandAndWrite(cnx, consumer.consumerId, 0L, 0L, - null, ackType, null, true, null, entriesToAck); + null, ackType, null, true, null, entriesToAck, null); } else { // if don't support multi message ack, it also support ack receipt, so we should not think about the // ack receipt in this logic for (MessageIdImpl cMsgId : chunkMsgIds) { newMessageAckCommandAndWrite(cnx, consumerId, cMsgId.getLedgerId(), cMsgId.getEntryId(), - bitSet, ackType, map, true, null, null); + bitSet, ackType, map, true, null, null, null); } completableFuture = CompletableFuture.completedFuture(null); } } else { completableFuture = newMessageAckCommandAndWrite(cnx, consumerId, msgId.getLedgerId(), msgId.getEntryId(), - bitSet, ackType, map, true, null, null); + bitSet, ackType, map, true, null, null, null); } return completableFuture; } @@ -569,7 +584,8 @@ private CompletableFuture newMessageAckCommandAndWrite( long entryId, BitSetRecyclable ackSet, AckType ackType, Map properties, boolean flush, TimedCompletableFuture timedCompletableFuture, - List> entriesToAck) { + List> entriesToAck, + Runnable writeFailureCallback) { if (consumer.isAckReceiptEnabled()) { final long requestId = consumer.getClient().newRequestId(); final ByteBuf cmd; @@ -611,14 +627,53 @@ private CompletableFuture newMessageAckCommandAndWrite( cmd = Commands.newMultiMessageAck(consumerId, entriesToAck, -1); } if (flush) { - cnx.ctx().writeAndFlush(cmd, cnx.ctx().voidPromise()); + if (writeFailureCallback == null) { + cnx.ctx().writeAndFlush(cmd, cnx.ctx().voidPromise()); + } else { + cnx.ctx().writeAndFlush(cmd).addListener(writeFuture -> { + if (!writeFuture.isSuccess()) { + writeFailureCallback.run(); + } + }); + } } else { - cnx.ctx().write(cmd, cnx.ctx().voidPromise()); + if (writeFailureCallback == null) { + cnx.ctx().write(cmd, cnx.ctx().voidPromise()); + } else { + cnx.ctx().write(cmd).addListener(writeFuture -> { + if (!writeFuture.isSuccess()) { + writeFailureCallback.run(); + } + }); + } } return CompletableFuture.completedFuture(null); } } + private void restoreIndividualAck(MessageIdAdv messageId, @Nullable MessageIdImpl[] chunkMsgIds) { + pendingIndividualAcks.add(messageId); + restoreChunkedMessageIds(messageId, chunkMsgIds); + } + + private void restoreIndividualAndBatchIndexAcks(List messageIds, + Map chunkMsgIds, + List> batchIndexAcks) { + pendingIndividualAcks.addAll(messageIds); + chunkMsgIds.forEach(this::restoreChunkedMessageIds); + batchIndexAcks.forEach(entry -> pendingIndividualBatchIndexAcks.merge(entry.getKey(), entry.getValue(), + (currentValue, valueToRestore) -> { + currentValue.and(valueToRestore); + return currentValue; + })); + } + + private void restoreChunkedMessageIds(MessageIdAdv messageId, @Nullable MessageIdImpl[] chunkMsgIds) { + if (chunkMsgIds != null) { + consumer.unAckedChunkedMessageIdSequenceMap.putIfAbsent(messageId, chunkMsgIds); + } + } + public Optional acquireReadLock() { Optional optionalLock = Optional.ofNullable(consumer.isAckReceiptEnabled() ? lock.readLock() : null); optionalLock.ifPresent(Lock::lock); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java index 76f2f45a0304c..410819ef732b6 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.client.impl; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -28,9 +29,13 @@ import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.EventLoopGroup; import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.util.ReferenceCountUtil; +import io.netty.util.concurrent.Future; +import io.netty.util.concurrent.GenericFutureListener; import java.util.ArrayList; import java.util.BitSet; import java.util.Collections; @@ -46,6 +51,7 @@ import org.apache.pulsar.client.util.TimedCompletableFuture; import org.apache.pulsar.common.api.proto.CommandAck.AckType; import org.apache.pulsar.common.api.proto.ProtocolVersion; +import org.apache.pulsar.common.util.FutureUtil; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; @@ -57,6 +63,8 @@ public class AcknowledgementsGroupingTrackerTest { private ConsumerImpl consumer; private EventLoopGroup eventLoopGroup; private AtomicBoolean returnCnx = new AtomicBoolean(true); + private ChannelHandlerContext successCtx; + private AtomicBoolean failAckCommandSend = new AtomicBoolean(false); @BeforeClass public void setup() throws NoSuchFieldException, IllegalAccessException { @@ -71,9 +79,9 @@ public void setup() throws NoSuchFieldException, IllegalAccessException { doReturn(new ConsumerStatsRecorderImpl()).when(consumer).getStats(); doReturn(UnAckedMessageTracker.UNACKED_MESSAGE_TRACKER_DISABLED) .when(consumer).getUnAckedMessageTracker(); - ChannelHandlerContext ctx = ClientTestFixtures.mockChannelHandlerContext(); + successCtx = ClientTestFixtures.mockChannelHandlerContext(); doAnswer(invocation -> returnCnx.get() ? cnx : null).when(consumer).getClientCnx(); - doReturn(ctx).when(cnx).ctx(); + doReturn(successCtx).when(cnx).ctx(); } @DataProvider(name = "isNeedReceipt") @@ -324,6 +332,60 @@ public void testAckTrackerMultiAck(boolean isNeedReceipt) { tracker.close(); } + @Test + public void testFlushRetainsPendingIndividualAckOnSendFailureWithoutAckReceipt() throws Exception { + ConsumerConfigurationData conf = new ConsumerConfigurationData<>(); + conf.setAcknowledgementsGroupTimeMicros(TimeUnit.SECONDS.toMicros(10)); + conf.setAckReceiptEnabled(false); + doReturn(false).when(consumer).isAckReceiptEnabled(); + PersistentAcknowledgmentsGroupingTracker tracker = + new PersistentAcknowledgmentsGroupingTracker(consumer, conf, eventLoopGroup); + + MessageIdImpl msg1 = new MessageIdImpl(5, 1, 0); + tracker.addAcknowledgment(msg1, AckType.Individual, Collections.emptyMap()); + assertEquals(tracker.getPendingIndividualAcksSize(), 1); + + doReturn(createFailedChannelHandlerContext()).when(cnx).ctx(); + + tracker.flush(); + + assertTrue(tracker.isDuplicate(msg1)); + assertEquals(tracker.getPendingIndividualAcksSize(), 1); + + doReturn(successCtx).when(cnx).ctx(); + + tracker.flush(); + + assertFalse(tracker.isDuplicate(msg1)); + assertEquals(tracker.getPendingIndividualAcksSize(), 0); + tracker.close(); + } + + @Test + public void testFlushFailsAckFutureOnSendFailureWithAckReceipt() throws Exception { + ConsumerConfigurationData conf = new ConsumerConfigurationData<>(); + conf.setAcknowledgementsGroupTimeMicros(TimeUnit.SECONDS.toMicros(10)); + conf.setAckReceiptEnabled(true); + doReturn(true).when(consumer).isAckReceiptEnabled(); + PersistentAcknowledgmentsGroupingTracker tracker = + new PersistentAcknowledgmentsGroupingTracker(consumer, conf, eventLoopGroup); + + MessageIdImpl msg1 = new MessageIdImpl(5, 1, 0); + CompletableFuture ackFuture = + tracker.addAcknowledgment(msg1, AckType.Individual, Collections.emptyMap()); + assertEquals(tracker.getPendingIndividualAcksSize(), 1); + + failAckCommandSend.set(true); + tracker.flush(); + + assertTrue(ackFuture.isCompletedExceptionally()); + assertFalse(tracker.isDuplicate(msg1)); + assertEquals(tracker.getPendingIndividualAcksSize(), 0); + + failAckCommandSend.set(false); + tracker.close(); + } + @Test(dataProvider = "isNeedReceipt") public void testBatchAckTrackerMultiAck(boolean isNeedReceipt) throws Exception { ConsumerConfigurationData conf = new ConsumerConfigurationData<>(); @@ -464,12 +526,40 @@ public ClientCnxTest(ClientConfigurationData conf, EventLoopGroup eventLoopGroup @Override public CompletableFuture newAckForReceipt(ByteBuf request, long requestId) { + if (failAckCommandSend.get()) { + return FutureUtil.failedFuture(new RuntimeException("ack send failed")); + } return CompletableFuture.completedFuture(null); } @Override public void newAckForReceiptWithFuture(ByteBuf request, long requestId, TimedCompletableFuture future) { + if (failAckCommandSend.get()) { + future.completeExceptionally(new RuntimeException("ack send failed")); + } } } + + private ChannelHandlerContext createFailedChannelHandlerContext() { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + ChannelFuture listenerFuture = mock(ChannelFuture.class); + ChannelFuture failedFuture = mock(ChannelFuture.class); + when(failedFuture.isSuccess()).thenReturn(false); + when(failedFuture.cause()).thenReturn(new RuntimeException("ack send failed")); + doAnswer(invocation -> { + GenericFutureListener> listener = invocation.getArgument(0); + listener.operationComplete(failedFuture); + return listenerFuture; + }).when(listenerFuture).addListener(any()); + doAnswer(invocation -> { + ReferenceCountUtil.release(invocation.getArgument(0)); + return listenerFuture; + }).when(ctx).write(any()); + doAnswer(invocation -> { + ReferenceCountUtil.release(invocation.getArgument(0)); + return listenerFuture; + }).when(ctx).writeAndFlush(any()); + return ctx; + } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxRequestTimeoutQueueTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxRequestTimeoutQueueTest.java index c4dab1ad351ab..7e5d98b136a40 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxRequestTimeoutQueueTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxRequestTimeoutQueueTest.java @@ -64,6 +64,7 @@ void setupClientCnx() throws Exception { ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); Channel channel = mock(Channel.class); when(ctx.writeAndFlush(any())).thenAnswer(args -> mock(ChannelFuture.class)); + when(ctx.write(any())).thenAnswer(args -> mock(ChannelFuture.class)); when(ctx.channel()).thenReturn(channel); when(channel.remoteAddress()).thenReturn(new InetSocketAddress(1234)); cnx.channelActive(ctx); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientTestFixtures.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientTestFixtures.java index ae0797fa4939a..494dea365d9e6 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientTestFixtures.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientTestFixtures.java @@ -147,6 +147,11 @@ public static ChannelHandlerContext mockChannelHandlerContext() { }).when(listenerFuture).addListener(any()); // handle write and writeAndFlush methods so that the input message is released + doAnswer(invocation -> { + Object msg = invocation.getArgument(0); + ReferenceCountUtil.release(msg); + return listenerFuture; + }).when(ctx).write(any()); doAnswer(invocation -> { Object msg = invocation.getArgument(0); ReferenceCountUtil.release(msg); From ecd51a27d4870339a69f963193baad7fce93a6c9 Mon Sep 17 00:00:00 2001 From: zhenJiangWang Date: Wed, 29 Apr 2026 10:47:18 +0800 Subject: [PATCH 015/213] [fix][broker] Fix precision loss in DataSketchesSummaryLogger by replacing LongAdder with DoubleAdder for sum accumulation (#25594) (cherry picked from commit 00577a5438bbb3d959e5ca2ea037e760219b95d8) --- .../prometheus/metrics/DataSketchesSummaryLogger.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/DataSketchesSummaryLogger.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/DataSketchesSummaryLogger.java index 42c189d4bf3a7..7495f057aa007 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/DataSketchesSummaryLogger.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/DataSketchesSummaryLogger.java @@ -22,6 +22,7 @@ import com.yahoo.sketches.quantiles.DoublesUnion; import com.yahoo.sketches.quantiles.DoublesUnionBuilder; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.DoubleAdder; import java.util.concurrent.atomic.LongAdder; public class DataSketchesSummaryLogger { @@ -37,7 +38,7 @@ public class DataSketchesSummaryLogger { */ private volatile DoublesSketch values; private final LongAdder countAdder = new LongAdder(); - private final LongAdder sumAdder = new LongAdder(); + private final DoubleAdder sumAdder = new DoubleAdder(); public DataSketchesSummaryLogger() { this.current = new ThreadLocalAccessor(); @@ -48,7 +49,7 @@ public void registerEvent(long eventLatency, TimeUnit unit) { double valueMillis = unit.toMicros(eventLatency) / 1000.0; countAdder.increment(); - sumAdder.add((long) valueMillis); + sumAdder.add(valueMillis); current.getLocalData().updateSuccess(valueMillis); } @@ -69,7 +70,7 @@ public long getCount() { return countAdder.sum(); } - public long getSum() { + public double getSum() { return sumAdder.sum(); } From 462ee2b51944b59333bc517ff82d5f1fec0110b2 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Thu, 30 Apr 2026 06:18:54 -0700 Subject: [PATCH 016/213] [fix][broker] Fix stuck chunks in SharedConsumerAssignor permit tracking (#25620) (cherry picked from commit 759a5f520f81ba45caef9bd94a79997855a695fd) --- .../pulsar/broker/service/SharedConsumerAssignor.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SharedConsumerAssignor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SharedConsumerAssignor.java index bbf8dfd2b10fc..a317ad7560b23 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SharedConsumerAssignor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SharedConsumerAssignor.java @@ -89,7 +89,7 @@ public Map> assign(final List if (metadata == null || !metadata.hasUuid() || !metadata.hasChunkId() || !metadata.hasNumChunksFromMsg()) { consumerToEntries.computeIfAbsent(consumer, __ -> new ArrayList<>()).add(entryAndMetadata); } else { - final Consumer consumerForUuid = getConsumerForUuid(metadata, consumer, availablePermits); + final Consumer consumerForUuid = getConsumerForUuid(metadata, consumer); if (consumerForUuid == null) { unassignedMessageProcessor.accept(entryAndMetadata); continue; @@ -120,9 +120,7 @@ private Consumer getConsumer(final int numConsumers) { return null; } - private Consumer getConsumerForUuid(final MessageMetadata metadata, - final Consumer defaultConsumer, - final int currentAvailablePermits) { + private Consumer getConsumerForUuid(final MessageMetadata metadata, final Consumer defaultConsumer) { final String uuid = metadata.getUuid(); Consumer consumer = uuidToConsumer.get(uuid); if (consumer == null) { @@ -141,7 +139,9 @@ private Consumer getConsumerForUuid(final MessageMetadata metadata, // The last chunk is received, we should remove the cache uuidToConsumer.remove(uuid); } - consumerToPermits.put(consumer, currentAvailablePermits - 1); + // Decrement target consumer's permits, not the loop's local availablePermits — on a cache + // redirect those track different consumers. + consumerToPermits.put(consumer, permits - 1); return consumer; } } From 7ff3067673287fb1e78d54cff1f78de28e5e1f3c Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 23 Mar 2026 17:11:17 +0200 Subject: [PATCH 017/213] [fix][client] Fix stale Healthy state in SameAuthParamsLookupAutoClusterFailover causing flaky test (#25388) Co-authored-by: Claude Opus 4.6 (1M context) (cherry picked from commit 3bc834f2fa8aae1ffe8f2fb3f795acd3d0de755d) --- ...meAuthParamsLookupAutoClusterFailover.java | 5 + ...thParamsLookupAutoClusterFailoverTest.java | 254 ++++++++++++++++++ 2 files changed, 259 insertions(+) create mode 100644 pulsar-client/src/test/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailoverTest.java diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java index 743f2e15164ff..5ae7b96906e8c 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java @@ -143,6 +143,11 @@ private int findFailoverTo() { for (int i = currentPulsarServiceIndex + 1; i < pulsarServiceUrlArray.length; i++) { if (probeAvailable(i)) { return i; + } else { + // Mark the service as Failed to prevent a spurious recovery to it + // after we failover to a higher-indexed service. + pulsarServiceStateArray[i] = PulsarServiceState.Failed; + checkCounterArray[i].setValue(0); } } return -1; diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailoverTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailoverTest.java new file mode 100644 index 0000000000000..31e2bf6aaeda4 --- /dev/null +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailoverTest.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import io.netty.channel.EventLoopGroup; +import java.lang.reflect.Method; +import java.net.InetSocketAddress; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.mutable.MutableInt; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.pulsar.client.impl.SameAuthParamsLookupAutoClusterFailover.PulsarServiceState; +import org.apache.pulsar.client.util.ExecutorProvider; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.netty.EventLoopUtil; +import org.awaitility.Awaitility; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Test(groups = "broker-impl") +public class SameAuthParamsLookupAutoClusterFailoverTest { + + private static final String URL0 = "pulsar://broker0:6650"; + private static final String URL1 = "pulsar://broker1:6650"; + private static final String URL2 = "pulsar://broker2:6650"; + + private EventLoopGroup executor; + private PulsarClientImpl mockClient; + private SameAuthParamsLookupAutoClusterFailover failover; + private PulsarServiceState[] stateArray; + private MutableInt[] counterArray; + + @BeforeMethod + public void setup() throws Exception { + executor = EventLoopUtil.newEventLoopGroup(1, false, + new ExecutorProvider.ExtendedThreadFactory("test-failover")); + + String[] urlArray = new String[]{URL0, URL1, URL2}; + failover = SameAuthParamsLookupAutoClusterFailover.builder() + .pulsarServiceUrlArray(urlArray) + .failoverThreshold(1) + .recoverThreshold(2) + .checkHealthyIntervalMs(100) + .testTopic("a/b/c") + .markTopicNotFoundAsAvailable(true) + .build(); + + mockClient = mock(PulsarClientImpl.class); + doNothing().when(mockClient).updateServiceUrl(anyString()); + doNothing().when(mockClient).reloadLookUp(); + + FieldUtils.writeField(failover, "pulsarClient", mockClient, true); + FieldUtils.writeField(failover, "executor", executor, true); + + stateArray = (PulsarServiceState[]) FieldUtils.readField(failover, "pulsarServiceStateArray", true); + counterArray = (MutableInt[]) FieldUtils.readField(failover, "checkCounterArray", true); + } + + @AfterMethod(alwaysRun = true) + public void cleanup() { + if (executor != null) { + executor.shutdownGracefully(0, 0, TimeUnit.MILLISECONDS); + } + } + + private void setLookupResult(String url, boolean available) { + LookupService lookup = mock(LookupService.class); + if (available) { + InetSocketAddress addr = InetSocketAddress.createUnresolved("broker", 6650); + when(lookup.getBroker(any())) + .thenReturn(CompletableFuture.completedFuture( + new LookupTopicResult(addr, addr, false))); + } else { + when(lookup.getBroker(any())) + .thenReturn(FutureUtil.failedFuture( + new RuntimeException("connection refused"))); + } + when(mockClient.getLookup(url)).thenReturn(lookup); + } + + /** + * Reproduces the race condition where findFailoverTo() skips over an unavailable service + * without marking it as Failed. This leaves a stale Healthy state that causes a spurious + * recovery bounce (0 -> 2 -> 1 -> 2) instead of a clean failover (0 -> 2). + * + *

Before the fix, after failover from index 0 to 2, state[1] remained Healthy (stale). + * On the next check cycle, firstHealthyPulsarService() would see state[1]=Healthy and + * immediately "recover" to index 1 — which is actually a broken service. This caused + * unnecessary bouncing and, combined with 3-second probe timeouts on dead services, + * could push the total failover time past the test's awaitility timeout. + */ + @Test(timeOut = 30000) + public void testFindFailoverToMarksSkippedServicesAsFailed() throws Exception { + // url0 is down, url1 is down, url2 is healthy. + setLookupResult(URL0, false); + setLookupResult(URL1, false); + setLookupResult(URL2, true); + + // Pre-set state[0] to Failed (as if checkPulsarServices already detected it), + // then run one check cycle. All on the executor to ensure thread safety. + runOnExecutor(() -> { + stateArray[0] = PulsarServiceState.Failed; + counterArray[0].setValue(0); + }); + runCheckCycle(); + + // After the fix, findFailoverTo marks url1 as Failed when it fails probing. + // Verify on the executor thread where state is owned. + runOnExecutor(() -> { + assertEquals(failover.getCurrentPulsarServiceIndex(), 2, + "Should have failed over to index 2"); + assertEquals(stateArray[1], PulsarServiceState.Failed, + "Service 1 should be marked Failed by findFailoverTo, not remain stale Healthy"); + assertEquals(stateArray[2], PulsarServiceState.Healthy, + "Service 2 should remain Healthy"); + }); + } + + /** + * Verifies no spurious recovery bounce occurs after failover. Without the fix, + * the first check cycle after failover to index 2 would see stale Healthy state[1] + * and immediately switch to index 1. + */ + @Test(timeOut = 30000) + public void testNoSpuriousRecoveryBounceAfterFailover() throws Exception { + // url0 is down, url1 is down, url2 is healthy. + setLookupResult(URL0, false); + setLookupResult(URL1, false); + setLookupResult(URL2, true); + + // Pre-set state[0] to Failed. + runOnExecutor(() -> { + stateArray[0] = PulsarServiceState.Failed; + counterArray[0].setValue(0); + }); + + // Failover: 0 -> 2. + runCheckCycle(); + runOnExecutor(() -> assertEquals(failover.getCurrentPulsarServiceIndex(), 2)); + + // Run another check cycle. Without the fix, state[1] would be stale Healthy, + // and firstHealthyPulsarService would return 1, causing a spurious switch. + runCheckCycle(); + runOnExecutor(() -> assertEquals(failover.getCurrentPulsarServiceIndex(), 2, + "Should stay at index 2, not bounce to index 1")); + } + + /** + * Verifies that recovery still works correctly for a service that was marked Failed + * by findFailoverTo, once that service becomes available again. + */ + @Test(timeOut = 30000) + public void testRecoveryAfterFindFailoverToMarksServiceFailed() throws Exception { + // url0 is down, url1 is down, url2 is healthy. + setLookupResult(URL0, false); + setLookupResult(URL1, false); + setLookupResult(URL2, true); + + // Pre-set state[0] to Failed and trigger failover 0 -> 2. + runOnExecutor(() -> { + stateArray[0] = PulsarServiceState.Failed; + counterArray[0].setValue(0); + }); + runCheckCycle(); + runOnExecutor(() -> { + assertEquals(failover.getCurrentPulsarServiceIndex(), 2); + assertEquals(stateArray[1], PulsarServiceState.Failed); + }); + + // Now make url1 healthy (simulating recovery of that service). + setLookupResult(URL1, true); + + // Run check cycles until service 1 recovers. + // Failed -> PreRecover (1 check) -> Healthy (recoverThreshold=2, so 1 more check). + Awaitility.await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> { + runCheckCycle(); + runOnExecutor(() -> { + assertEquals(failover.getCurrentPulsarServiceIndex(), 1, + "Should recover to index 1 after it becomes healthy"); + assertEquals(stateArray[1], PulsarServiceState.Healthy); + }); + }); + } + + private void runOnExecutor(Runnable task) throws Exception { + try { + executor.submit(task).get(5, TimeUnit.SECONDS); + } catch (java.util.concurrent.ExecutionException e) { + // Unwrap so that AssertionErrors propagate directly to Awaitility. + if (e.getCause() instanceof AssertionError) { + throw (AssertionError) e.getCause(); + } + throw e; + } + } + + private void runCheckCycle() throws Exception { + runOnExecutor(() -> { + try { + Method checkMethod = SameAuthParamsLookupAutoClusterFailover.class + .getDeclaredMethod("checkPulsarServices"); + checkMethod.setAccessible(true); + Method firstHealthyMethod = SameAuthParamsLookupAutoClusterFailover.class + .getDeclaredMethod("firstHealthyPulsarService"); + firstHealthyMethod.setAccessible(true); + Method findFailoverMethod = SameAuthParamsLookupAutoClusterFailover.class + .getDeclaredMethod("findFailoverTo"); + findFailoverMethod.setAccessible(true); + Method updateMethod = SameAuthParamsLookupAutoClusterFailover.class + .getDeclaredMethod("updateServiceUrl", int.class); + updateMethod.setAccessible(true); + + checkMethod.invoke(failover); + int firstHealthy = (int) firstHealthyMethod.invoke(failover); + int currentIndex = failover.getCurrentPulsarServiceIndex(); + if (firstHealthy != currentIndex) { + if (firstHealthy < 0) { + int failoverTo = (int) findFailoverMethod.invoke(failover); + if (failoverTo >= 0) { + updateMethod.invoke(failover, failoverTo); + } + } else { + updateMethod.invoke(failover, firstHealthy); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } +} From 7b9afebc1b3a272a6faa0015ac02f0fea4f6f8e0 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Wed, 20 May 2026 06:48:58 -0700 Subject: [PATCH 018/213] [fix][client] Reset higher-index states on recovery in SameAuthParamsLookupAutoClusterFailover (#25826) (cherry picked from commit 4229e197479e8ea10164c30abf95c41ceff2b344) --- ...meAuthParamsLookupAutoClusterFailover.java | 11 ++++ ...thParamsLookupAutoClusterFailoverTest.java | 63 +++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java index 5ae7b96906e8c..448dd95c4fdd9 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java @@ -261,6 +261,17 @@ private void updateServiceUrl(int targetIndex) { try { pulsarClient.updateServiceUrl(targetUrl); pulsarClient.reloadLookUp(); + // When recovering to a higher-priority service, the check loop will only probe + // indices 0..targetIndex going forward. Any transient state (e.g., PreFail from + // a single timed-out probe) at higher indices would become stuck because those + // indices are no longer probed. Reset them so they start fresh if a future + // failover needs to consider them again. + if (targetIndex < currentPulsarServiceIndex) { + for (int i = targetIndex + 1; i < pulsarServiceStateArray.length; i++) { + pulsarServiceStateArray[i] = PulsarServiceState.Healthy; + checkCounterArray[i].setValue(0); + } + } currentPulsarServiceIndex = targetIndex; } catch (Exception e) { log.error("Failed to {}", logMsg, e); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailoverTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailoverTest.java index 31e2bf6aaeda4..959d913dff892 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailoverTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailoverTest.java @@ -168,6 +168,69 @@ public void testNoSpuriousRecoveryBounceAfterFailover() throws Exception { "Should stay at index 2, not bounce to index 1")); } + /** + * Reproduces the bug fixed by resetting state of higher-priority-than-target indices on + * recovery. The check loop only probes indices 0..currentPulsarServiceIndex, so if a + * higher index was left in a transient state (e.g., PreFail from a single timed-out + * probe) at the moment of recovery, it would stay there forever because no future probe + * ever visits it. + * + *

Scenario: failover 0 -> 2 (state[2]=Healthy). url1 recovers and is about to trigger + * recovery 2 -> 1. On the same check cycle that promotes state[1] to Healthy, url2 sees + * one transient probe failure that flips state[2] from Healthy to PreFail. The recovery + * fires (current 2 -> 1) and from that point on the loop only probes indices 0 and 1. + * + *

Without the fix, state[2] stays at PreFail forever. With the fix, state[2] is + * reset to Healthy when the recovery transition runs. + */ + @Test(timeOut = 30000) + public void testRecoveryResetsHigherIndexStaleState() throws Exception { + // url0 down, url1 down, url2 up. + setLookupResult(URL0, false); + setLookupResult(URL1, false); + setLookupResult(URL2, true); + + // Pre-set state[0]=Failed and trigger failover 0 -> 2. + runOnExecutor(() -> { + stateArray[0] = PulsarServiceState.Failed; + counterArray[0].setValue(0); + }); + runCheckCycle(); + runOnExecutor(() -> { + assertEquals(failover.getCurrentPulsarServiceIndex(), 2); + assertEquals(stateArray[1], PulsarServiceState.Failed); + assertEquals(stateArray[2], PulsarServiceState.Healthy); + }); + + // url1 becomes healthy; first check cycle moves state[1] Failed -> PreRecover. + setLookupResult(URL1, true); + runCheckCycle(); + runOnExecutor(() -> { + assertEquals(failover.getCurrentPulsarServiceIndex(), 2); + assertEquals(stateArray[1], PulsarServiceState.PreRecover); + assertEquals(stateArray[2], PulsarServiceState.Healthy); + }); + + // On the cycle that completes recovery (state[1] PreRecover -> Healthy and triggers + // updateServiceUrl(1)), inject a single failed probe at url2 so state[2] flips + // Healthy -> PreFail just before the index transition. + setLookupResult(URL2, false); + runCheckCycle(); + + // After recovery to index 1: + // - With the fix: state[2] is reset to Healthy on the transition. + // - Without the fix: state[2] is stuck at PreFail forever — the check loop now + // only iterates 0..1 and never visits index 2 again. + runOnExecutor(() -> { + assertEquals(failover.getCurrentPulsarServiceIndex(), 1); + assertEquals(stateArray[1], PulsarServiceState.Healthy); + assertEquals(stateArray[2], PulsarServiceState.Healthy, + "state[2] should be reset to Healthy on recovery, not stuck at PreFail"); + assertEquals(counterArray[2].intValue(), 0, + "counter[2] should be reset to 0 on recovery"); + }); + } + /** * Verifies that recovery still works correctly for a service that was marked Failed * by findFailoverTo, once that service becomes available again. From 41a0dda2004ace5a24a08b8218eeb108cf214c3d Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 1 Jun 2026 23:59:29 +0300 Subject: [PATCH 019/213] [fix][test][branch-4.0] Fix PersistentMessageExpiryMonitorTest - master branch contains this change - the second parameter will also match a null value --- .../mledger/impl/PersistentMessageExpiryMonitorTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/bookkeeper/mledger/impl/PersistentMessageExpiryMonitorTest.java b/pulsar-broker/src/test/java/org/apache/bookkeeper/mledger/impl/PersistentMessageExpiryMonitorTest.java index 39aec66726e32..e487d4c1c8a22 100644 --- a/pulsar-broker/src/test/java/org/apache/bookkeeper/mledger/impl/PersistentMessageExpiryMonitorTest.java +++ b/pulsar-broker/src/test/java/org/apache/bookkeeper/mledger/impl/PersistentMessageExpiryMonitorTest.java @@ -24,7 +24,6 @@ import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.spy; import static org.testng.AssertJUnit.assertEquals; -import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -100,7 +99,7 @@ void testConcurrentlyExpireMessages() throws Exception { } }); return true; - }).when(spyCursor).asyncMarkDelete(any(Position.class), any(Map.class), + }).when(spyCursor).asyncMarkDelete(any(Position.class), any(), any(AsyncCallbacks.MarkDeleteCallback.class), any()); doAnswer(invocationOnMock -> { calledFindPositionCount.incrementAndGet(); From b06d879f8d48291af11f4b5ffb54f2d28e642bdc Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Thu, 19 Mar 2026 18:08:38 -0700 Subject: [PATCH 020/213] [fix][test] Fix flaky MessagePublishBufferThrottleTest.testBlockByPublishRateLimiting (#25365) (cherry picked from commit a3ae70545f3aef40e8f5cfcced907e6500b08894) --- .../MessagePublishBufferThrottleTest.java | 60 ++++++++++++------- 1 file changed, 39 insertions(+), 21 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessagePublishBufferThrottleTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessagePublishBufferThrottleTest.java index 0faae14da08ba..6a74df3eecc00 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessagePublishBufferThrottleTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessagePublishBufferThrottleTest.java @@ -20,10 +20,9 @@ import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat; import static org.apache.pulsar.broker.stats.BrokerOpenTelemetryTestUtil.assertMetricLongSumValue; -import static org.testng.Assert.fail; -import java.util.concurrent.CompletableFuture; +import io.opentelemetry.sdk.metrics.data.MetricData; +import java.util.Collection; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import lombok.Cleanup; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.testcontext.PulsarTestContext; @@ -134,33 +133,32 @@ public void testBlockByPublishRateLimiting() throws Exception { pulsarTestContext.getMockBookKeeper().addEntryDelay(5, TimeUnit.SECONDS); - // Block by publish buffer. + // Block by publish buffer: 10 x 1MB messages with a 1MB buffer limit. byte[] payload = new byte[1024 * 1024]; for (int i = 0; i < 10; i++) { producer.sendAsync(payload); } - Awaitility.await().untilAsserted(() -> assertRateLimitCounter(ConnectionRateLimitOperationName.PAUSED, 1)); + // Wait for at least one pause event to be recorded. + Awaitility.await().untilAsserted( + () -> assertRateLimitCounterAtLeast(ConnectionRateLimitOperationName.PAUSED, 1)); - CompletableFuture flushFuture = producer.flushAsync(); + // Verify that no resume has happened yet while messages are still blocked. + Awaitility.await().untilAsserted( + () -> assertRateLimitCounter(ConnectionRateLimitOperationName.RESUMED, 0)); - // Block by publish rate. - // After 1 second, the message buffer throttling will be lifted, but the rate limiting will still be in place. - assertRateLimitCounter(ConnectionRateLimitOperationName.PAUSED, 1); - assertRateLimitCounter(ConnectionRateLimitOperationName.RESUMED, 0); - - try { - flushFuture.get(2, TimeUnit.SECONDS); - fail("Should have timed out"); - } catch (TimeoutException e) { - // Ok - } - - flushFuture.join(); + // Flush and wait for all messages to complete. + producer.flush(); + // After all messages are sent, the number of pauses and resumes should match: + // every pause must eventually be followed by a resume. Awaitility.await().untilAsserted(() -> { - assertRateLimitCounter(ConnectionRateLimitOperationName.PAUSED, 10); - assertRateLimitCounter(ConnectionRateLimitOperationName.RESUMED, 10); + var metrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + long pausedCount = getMetricLongSumValue(metrics, ConnectionRateLimitOperationName.PAUSED); + long resumedCount = getMetricLongSumValue(metrics, ConnectionRateLimitOperationName.RESUMED); + Assert.assertTrue(pausedCount > 0, "Expected at least one pause event"); + Assert.assertEquals(pausedCount, resumedCount, + "Paused and resumed counts should match after all messages are sent"); }); } @@ -206,4 +204,24 @@ private void assertRateLimitCounter(ConnectionRateLimitOperationName connectionR connectionRateLimitState.attributes, expectedCount); } } + + private void assertRateLimitCounterAtLeast(ConnectionRateLimitOperationName connectionRateLimitState, + int minExpectedCount) { + var metrics = pulsarTestContext.getOpenTelemetryMetricReader().collectAllMetrics(); + assertMetricLongSumValue(metrics, BrokerService.CONNECTION_RATE_LIMIT_COUNT_METRIC_NAME, + connectionRateLimitState.attributes, + actual -> assertThat(actual).isGreaterThanOrEqualTo(minExpectedCount)); + } + + private long getMetricLongSumValue(Collection metrics, + ConnectionRateLimitOperationName connectionRateLimitState) { + var attributesMap = connectionRateLimitState.attributes.asMap(); + return metrics.stream() + .filter(m -> m.getName().equals(BrokerService.CONNECTION_RATE_LIMIT_COUNT_METRIC_NAME)) + .flatMap(m -> m.getLongSumData().getPoints().stream()) + .filter(point -> point.getAttributes().asMap().equals(attributesMap)) + .mapToLong(io.opentelemetry.sdk.metrics.data.LongPointData::getValue) + .findFirst() + .orElse(0L); + } } From 118de6cbab2df0a96bdc0dd2a04319a952d6f86c Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 2 Jun 2026 09:38:02 +0300 Subject: [PATCH 021/213] [fix][test][branch-4.0] Fix PulsarFunctionTlsTest - resolve a previous incorrect merge conflict resolution --- .../apache/pulsar/functions/worker/PulsarFunctionTlsTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java index f2e49c4d7b164..7872b79566715 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java @@ -268,9 +268,6 @@ public void testFunctionsCreation() throws Exception { .ignoreExceptionsMatching(PulsarFunctionTlsTest::isLeaderNotReady) .untilAsserted(() -> createAdmin.functions() .createFunctionWithUrl(functionConfig, jarFilePathUrl)); - pulsarAdmins[i].functions().createFunctionWithUrl( - functionConfig, jarFilePathUrl - ); // Function creation is not strongly consistent, so this test can fail with a get that is too eager and // does not have retries. From ae547c687ad615922a04858b1a93c35ba7bd880d Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Mon, 30 Mar 2026 15:53:41 -0700 Subject: [PATCH 022/213] [fix][test] Fix flaky testMsgDropStat in NonPersistentTopicTest (#25426) (cherry picked from commit e9630c60e8bff865e1bfeb2ed26a6c1d65f040b4) --- .../client/api/NonPersistentTopicTest.java | 54 +++++++++---------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java index 4fb67b4123f1c..6b6b75a1e3f99 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java @@ -59,7 +59,6 @@ import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.impl.ConsumerImpl; -import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.client.impl.MultiTopicsConsumerImpl; import org.apache.pulsar.client.impl.PartitionedProducerImpl; import org.apache.pulsar.client.impl.ProducerImpl; @@ -886,30 +885,31 @@ public void testMsgDropStat() throws Exception { ExecutorService executor = Executors.newFixedThreadPool(threads); byte[] msgData = "testData".getBytes(); + NonPersistentTopic topic = + (NonPersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get(); + /* - * Trigger at least one publisher drop through concurrent send() calls. + * Send concurrent bursts until publisher AND subscription drop rates are all > 0. + * + * Each burst uses a CyclicBarrier so all threads send simultaneously. With + * maxConcurrentNonPersistentMessagePerConnection = 0, ServerCnx drops overlapping + * sends (publisher drops). Once subscriber queues (size 1) are full, the dispatcher + * also drops delivered messages (subscription drops). * - * Uses CyclicBarrier to ensure all threads send simultaneously, creating overlap. - * With maxConcurrentNonPersistentMessagePerConnection = 0, ServerCnx#handleSend - * drops any send while another is in-flight, returning MessageId with entryId = -1. - * Awaitility repeats whole bursts (bounded to 20s) until a drop is observed. + * IMPORTANT: updateRates() calls Rate.calculateRate() which resets counters via + * sumThenReset(). We must keep sending fresh bursts so each updateRates() call + * sees new drops, rather than retrying with stale (reset) counters. */ - AtomicBoolean publisherDropSeen = new AtomicBoolean(false); - Awaitility.await().atMost(Duration.ofSeconds(20)).until(() -> { + Awaitility.await().atMost(Duration.ofSeconds(20)).pollInterval(Duration.ofMillis(100)).until(() -> { CyclicBarrier barrier = new CyclicBarrier(threads); CountDownLatch completionLatch = new CountDownLatch(threads); AtomicReference error = new AtomicReference<>(); - publisherDropSeen.set(false); for (int i = 0; i < threads; i++) { executor.submit(() -> { try { barrier.await(); - MessageId msgId = producer.send(msgData); - // Publisher drop is signaled by MessageIdImpl.entryId == -1 - if (msgId instanceof MessageIdImpl && ((MessageIdImpl) msgId).getEntryId() == -1) { - publisherDropSeen.set(true); - } + producer.send(msgData); } catch (Throwable t) { if (t instanceof InterruptedException) { Thread.currentThread().interrupt(); @@ -921,27 +921,23 @@ public void testMsgDropStat() throws Exception { }); } - // Wait for all sends to complete. - assertTrue(completionLatch.await(20, TimeUnit.SECONDS)); - - assertNull(error.get(), "Concurrent send encountered an exception"); - return publisherDropSeen.get(); - }); - - assertTrue(publisherDropSeen.get(), "Expected at least one publisher drop (entryId == -1)"); - - NonPersistentTopic topic = - (NonPersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get(); + completionLatch.await(20, TimeUnit.SECONDS); + if (error.get() != null) { + return false; + } - Awaitility.await().ignoreExceptions().untilAsserted(() -> { pulsar.getBrokerService().updateRates(); NonPersistentTopicStats stats = topic.getStats(false, false, false); + if (stats.getPublishers().isEmpty()) { + return false; + } NonPersistentPublisherStats npStats = stats.getPublishers().get(0); NonPersistentSubscriptionStats sub1Stats = stats.getSubscriptions().get("subscriber-1"); NonPersistentSubscriptionStats sub2Stats = stats.getSubscriptions().get("subscriber-2"); - assertTrue(npStats.getMsgDropRate() > 0); - assertTrue(sub1Stats.getMsgDropRate() > 0); - assertTrue(sub2Stats.getMsgDropRate() > 0); + return sub1Stats != null && sub2Stats != null + && npStats.getMsgDropRate() > 0 + && sub1Stats.getMsgDropRate() > 0 + && sub2Stats.getMsgDropRate() > 0; }); } finally { From 3733db29d3177a8735929622e02f244a6bc2b9f3 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Tue, 10 Jun 2025 14:35:04 +0800 Subject: [PATCH 023/213] [improve][broker] PIP-380: Support-setting-up-specific-namespaces-to-skipping-the-load-shedding (#23549) (cherry picked from commit 0f9ea181b084907ec8cb3d25535f7c6e3d2ffdc2) --- .../pulsar/broker/ServiceConfiguration.java | 7 ++++ .../extensions/ExtensibleLoadManagerImpl.java | 28 ++++++++++++- .../extensions/models/TopKBundles.java | 11 +++++- .../extensions/scheduler/TransferShedder.java | 9 +++++ .../RoundRobinBrokerSelectionStrategy.java | 37 ++++++++++++++++++ .../impl/ModularLoadManagerImpl.java | 39 +++++++++++++++++-- .../ExtensibleLoadManagerImplTest.java | 35 +++++++++++++++++ .../extensions/models/TopKBundlesTest.java | 22 +++++++++++ .../scheduler/TransferShedderTest.java | 19 +++++++++ 9 files changed, 201 insertions(+), 6 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/RoundRobinBrokerSelectionStrategy.java diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 9314117bed5f5..9df20329db06a 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -3128,6 +3128,13 @@ public double getLoadBalancerBandwidthOutResourceWeight() { ) private boolean loadBalancerSheddingBundlesWithPoliciesEnabled = false; + @FieldContext( + dynamic = true, + category = CATEGORY_LOAD_BALANCER, + doc = "The namespaces to be excluded from load shedding" + ) + private Set loadBalancerSheddingExcludedNamespaces = new HashSet<>(); + @FieldContext( category = CATEGORY_LOAD_BALANCER, doc = "Time to wait before fixing any stuck in-flight service unit states. " diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index ef29d7d9a74f3..af95df60174db 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -84,6 +84,7 @@ import org.apache.pulsar.broker.loadbalance.extensions.strategy.BrokerSelectionStrategy; import org.apache.pulsar.broker.loadbalance.extensions.strategy.BrokerSelectionStrategyFactory; import org.apache.pulsar.broker.loadbalance.extensions.strategy.LeastResourceUsageWithWeight; +import org.apache.pulsar.broker.loadbalance.extensions.strategy.RoundRobinBrokerSelectionStrategy; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies; import org.apache.pulsar.broker.namespace.LookupOptions; @@ -161,6 +162,8 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager, BrokerS @Getter private final BrokerSelectionStrategy brokerSelectionStrategy; + private final BrokerSelectionStrategy sheddingExcludedNamespaceSelectionStrategy; + @Getter private final List brokerFilterPipeline; @@ -254,6 +257,7 @@ public ExtensibleLoadManagerImpl() { this.brokerFilterPipeline.add(new BrokerMaxTopicCountFilter()); this.brokerFilterPipeline.add(new BrokerVersionFilter()); this.brokerSelectionStrategy = createBrokerSelectionStrategy(); + this.sheddingExcludedNamespaceSelectionStrategy = new RoundRobinBrokerSelectionStrategy(); } public static boolean isLoadManagerExtensionEnabled(PulsarService pulsar) { @@ -636,11 +640,33 @@ public CompletableFuture> selectAsync(ServiceUnitId bundle, return Optional.empty(); } Set candidateBrokers = availableBrokerCandidates.keySet(); - return getBrokerSelectionStrategy().select(candidateBrokers, bundle, context); + return getBrokerSelectionStrategy(bundle).select(candidateBrokers, bundle, context); }); }); } + /** + * For shedding excluded namespaces, use RoundRobinBrokerSelector to assign the ownership, + * it can make the assignment more average because these will not automatically rebalance to + * another broker unless manually unloaded it. + * + * @param bundle the bundle to assign + * @return the broker selection strategy + */ + private BrokerSelectionStrategy getBrokerSelectionStrategy(ServiceUnitId bundle) { + + Set sheddingExcludedNamespaces = conf.getLoadBalancerSheddingExcludedNamespaces(); + + var namespace = NamespaceBundle.getBundleNamespace(bundle.toString()); + if (sheddingExcludedNamespaces.contains(namespace)) { + if (debug(conf, log)) { + log.info("Use round robin broker selector for {}", bundle); + } + return sheddingExcludedNamespaceSelectionStrategy; + } + return brokerSelectionStrategy; + } + @Override public CompletableFuture checkOwnershipAsync(Optional topic, ServiceUnitId bundleUnit) { return getOwnershipAsync(topic, bundleUnit) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java index 9c6e963417813..481e907d04439 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java @@ -22,6 +22,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; @@ -68,8 +69,10 @@ public TopKBundles(PulsarService pulsar) { public void update(Map bundleStats, int topk) { arr.clear(); try { + var conf = pulsar.getConfiguration(); var isLoadBalancerSheddingBundlesWithPoliciesEnabled = - pulsar.getConfiguration().isLoadBalancerSheddingBundlesWithPoliciesEnabled(); + conf.isLoadBalancerSheddingBundlesWithPoliciesEnabled(); + Set sheddingExcludedNamespaces = conf.getLoadBalancerSheddingExcludedNamespaces(); for (var etr : bundleStats.entrySet()) { String bundle = etr.getKey(); var stat = etr.getValue(); @@ -79,12 +82,16 @@ public void update(Map bundleStats, int topk) { continue; } // TODO: do not filter system topic while shedding - if (NamespaceService.isSystemServiceNamespace(NamespaceBundle.getBundleNamespace(bundle))) { + String namespace = NamespaceBundle.getBundleNamespace(bundle); + if (NamespaceService.isSystemServiceNamespace(namespace)) { continue; } if (!isLoadBalancerSheddingBundlesWithPoliciesEnabled && hasPolicies(bundle)) { continue; } + if (sheddingExcludedNamespaces.contains(namespace)) { + continue; + } arr.add(etr); } var topKBundlesLoadData = loadData.getTopBundlesLoadData(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java index b5255f2713a6a..18555bc18fac9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java @@ -493,6 +493,7 @@ public Set findBundlesForUnloading(LoadManagerContext context, } int remainingTopBundles = maxBrokerTopBundlesLoadData.size(); + Set sheddingExcludedNamespaces = conf.getLoadBalancerSheddingExcludedNamespaces(); for (var e : maxBrokerTopBundlesLoadData) { String bundle = e.bundleName(); if (channel != null && !channel.isOwner(bundle, maxBroker)) { @@ -502,6 +503,14 @@ public Set findBundlesForUnloading(LoadManagerContext context, } continue; } + final String namespaceName = NamespaceBundle.getBundleNamespace(bundle); + if (sheddingExcludedNamespaces.contains(namespaceName)) { + if (debugMode) { + log.info(String.format(CANNOT_UNLOAD_BUNDLE_MSG + + " Bundle namespace has been found in sheddingExcludedNamespaces", bundle)); + } + continue; + } if (recentlyUnloadedBundles.containsKey(bundle)) { if (debugMode) { log.info(String.format(CANNOT_UNLOAD_BUNDLE_MSG diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/RoundRobinBrokerSelectionStrategy.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/RoundRobinBrokerSelectionStrategy.java new file mode 100644 index 0000000000000..2f356ec1f5ec1 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/RoundRobinBrokerSelectionStrategy.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.strategy; + +import java.util.Optional; +import java.util.Set; +import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; +import org.apache.pulsar.broker.loadbalance.impl.RoundRobinBrokerSelector; +import org.apache.pulsar.common.naming.ServiceUnitId; + +/** + * Simple Round Robin Broker Selection Strategy. + */ +public class RoundRobinBrokerSelectionStrategy implements BrokerSelectionStrategy { + private final RoundRobinBrokerSelector selector = new RoundRobinBrokerSelector(); + + @Override + public Optional select(Set brokers, ServiceUnitId bundle, LoadManagerContext context) { + return selector.selectBroker(brokers, null, null, context.brokerConfiguration()); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java index 1f549cbb66ec7..a9d7ddd78e07d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java @@ -152,6 +152,8 @@ public class ModularLoadManagerImpl implements ModularLoadManager { // Strategy used to determine where new topics should be placed. private ModularLoadManagerStrategy placementStrategy; + private ModularLoadManagerStrategy sheddingExcludedNamespaceSelectionStrategy; + // Policies used to determine which brokers are available for particular namespaces. private SimpleResourceAllocationPolicies policies; @@ -252,6 +254,7 @@ public void initialize(final PulsarService pulsar) { defaultStats.msgRateOut = DEFAULT_MESSAGE_RATE; placementStrategy = ModularLoadManagerStrategy.create(conf); + sheddingExcludedNamespaceSelectionStrategy = new RoundRobinBrokerSelector(); policies = new SimpleResourceAllocationPolicies(pulsar); filterPipeline.add(new BrokerLoadManagerClassFilter()); filterPipeline.add(new BrokerVersionFilter()); @@ -641,6 +644,7 @@ public synchronized void doLoadShedding() { final Map recentlyUnloadedBundles = loadData.getRecentlyUnloadedBundles(); recentlyUnloadedBundles.keySet().removeIf(e -> recentlyUnloadedBundles.get(e) < timeout); + Set sheddingExcludedNamespaces = conf.getLoadBalancerSheddingExcludedNamespaces(); final Multimap bundlesToUnload = loadSheddingStrategy.findBundlesForUnloading(loadData, conf); bundlesToUnload.asMap().forEach((broker, bundles) -> { @@ -648,6 +652,13 @@ public synchronized void doLoadShedding() { bundles.forEach(bundle -> { final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundle); final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle); + if (sheddingExcludedNamespaces.contains(namespaceName)) { + if (log.isDebugEnabled()) { + log.debug("[{}] Skipping load shedding for namespace {}", + loadSheddingStrategy.getClass().getSimpleName(), namespaceName); + } + return; + } if (!shouldNamespacePoliciesUnload(namespaceName, bundleRange, broker)) { return; } @@ -931,8 +942,22 @@ Optional selectBroker(final ServiceUnitId serviceUnit) { brokerTopicLoadingPredicate); } - // Choose a broker among the potentially smaller filtered list, when possible - Optional broker = placementStrategy.selectBroker(brokerCandidateCache, data, loadData, conf); + Optional broker; + // For shedding excluded namespaces, use RoundRobinBrokerSelector to assign the ownership, + // it can make the assignment more average because these will not automatically rebalance to + // another broker unless manually unloaded it. + Set sheddingExcludedNamespaces = conf.getLoadBalancerSheddingExcludedNamespaces(); + String namespaceNameFromBundleName = LoadManagerShared.getNamespaceNameFromBundleName(bundle); + if (sheddingExcludedNamespaces.contains(namespaceNameFromBundleName)) { + if (log.isDebugEnabled()) { + log.debug("Use round robin broker selector for {}", bundle); + } + broker = sheddingExcludedNamespaceSelectionStrategy + .selectBroker(brokerCandidateCache, data, loadData, conf); + } else { + // Choose a broker among the potentially smaller filtered list, when possible + broker = placementStrategy.selectBroker(brokerCandidateCache, data, loadData, conf); + } if (log.isDebugEnabled()) { log.debug("Selected broker {} from candidate brokers {}", broker, brokerCandidateCache); } @@ -1139,7 +1164,15 @@ public void writeBrokerDataOnZooKeeper(boolean force) { */ private int selectTopKBundle() { bundleArr.clear(); - bundleArr.addAll(loadData.getBundleData().entrySet()); + Set sheddingExcludedNamespaces = conf.getLoadBalancerSheddingExcludedNamespaces(); + for (Map.Entry entry : loadData.getBundleData().entrySet()) { + String bundle = entry.getKey(); + String namespace = NamespaceBundle.getBundleNamespace(bundle); + if (sheddingExcludedNamespaces.contains(namespace)) { + continue; + } + bundleArr.add(entry); + } int maxNumberOfBundlesInBundleLoadReport = pulsar.getConfiguration() .getLoadBalancerMaxNumberOfBundlesInBundleLoadReport(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index 9792148800298..65d017499fdb0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -206,6 +206,41 @@ public void testAssign() throws Exception { assertEquals(webServiceUrl.get().toString(), brokerLookupData.get().getWebServiceUrl()); } + // Test that the load manager will use round-robin assignment + // if the namespace is in loadBalancerSheddingExcludedNamespaces. + @Test + public void testSelectBrokerForSheddingExcludedNamespaces() throws Exception { + pulsar1.getConfiguration().setLoadBalancerSheddingExcludedNamespaces(Set.of(defaultTestNamespace)); + try { + Pair topicAndBundle = + getBundleIsNotOwnByChangeEventTopic("test-topic" + UUID.randomUUID()); + NamespaceBundle bundle1 = topicAndBundle.getRight(); + Optional brokerLookupData1 = primaryLoadManager.assign(Optional.empty(), bundle1, + LookupOptions.builder().build()).get(); + assertTrue(brokerLookupData1.isPresent()); + log.info("Assign the bundle1 {} to {}", bundle1, brokerLookupData1); + + String webServiceUrl1 = brokerLookupData1.get().getWebServiceUrl(); + + Pair topicAndBundle2 = + getBundleIsNotOwnByChangeEventTopic("test-topic-" + UUID.randomUUID()); + + while (topicAndBundle2.getRight().toString().equals(topicAndBundle.getRight().toString()) + || primaryLoadManager.checkOwnershipAsync(Optional.empty(), topicAndBundle2.getRight()).get()) { + topicAndBundle2 = getBundleIsNotOwnByChangeEventTopic("test-topic-" + UUID.randomUUID()); + } + NamespaceBundle bundle2 = topicAndBundle2.getRight(); + Optional brokerLookupData2 = primaryLoadManager.assign(Optional.empty(), bundle2, + LookupOptions.builder().build()).get(); + assertTrue(brokerLookupData2.isPresent()); + log.info("Assign the bundle2 {} to {}", bundle2, brokerLookupData2); + String webServiceUrl2 = brokerLookupData2.get().getWebServiceUrl(); + assertNotEquals(webServiceUrl1, webServiceUrl2); + } finally { + pulsar1.getConfiguration().setLoadBalancerSheddingExcludedNamespaces(Set.of()); + } + } + @Test public void testLookupOptions() throws Exception { Pair topicAndBundle = diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java index 0f6ae9b2629c4..2be3108c63832 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundlesTest.java @@ -31,6 +31,7 @@ import java.util.Map; import java.util.Optional; import java.util.Random; +import java.util.Set; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; @@ -136,6 +137,27 @@ public void testSystemNamespace() { assertEquals(top0.bundleName(), bundle1); } + @Test + public void testSheddingExcludedNamespaces() { + Map bundleStats = new HashMap<>(); + var topKBundles = new TopKBundles(pulsar); + pulsar.getConfiguration().setLoadBalancerSheddingExcludedNamespaces(Set.of("my-tenant/my-namespace2")); + NamespaceBundleStats stats1 = new NamespaceBundleStats(); + stats1.msgRateIn = 500; + bundleStats.put("my-tenant/my-namespace2/0x00000000_0x0FFFFFFF", stats1); + + NamespaceBundleStats stats2 = new NamespaceBundleStats(); + stats2.msgRateIn = 10000; + stats2.msgThroughputOut = 10; + bundleStats.put(bundle1, stats2); + + topKBundles.update(bundleStats, 2); + + assertEquals(topKBundles.getLoadData().getTopBundlesLoadData().size(), 1); + var top0 = topKBundles.getLoadData().getTopBundlesLoadData().get(0); + assertEquals(top0.bundleName(), bundle1); + } + @Test public void testZeroMsgThroughputBundleStats() { Map bundleStats = new HashMap<>(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java index 716e1f2a2a280..a144d84fd7a59 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java @@ -621,6 +621,25 @@ public void testRecentlyUnloadedBundles() { assertEquals(counter.getLoadStd(), setupLoadStd, delta); } + @Test + public void testSheddingExcludedNamespaces() { + UnloadCounter counter = new UnloadCounter(); + TransferShedder transferShedder = new TransferShedder(counter); + var ctx = setupContext(); + ctx.brokerConfiguration().setLoadBalancerSheddingExcludedNamespaces( + Set.of("my-tenant/my-namespaceE", "my-tenant/my-namespaceD")); + + var res = transferShedder.findBundlesForUnloading(ctx, new HashMap<>(), Map.of()); + var expected = new HashSet(); + expected.add(new UnloadDecision(new Unload("broker3:8080", + "my-tenant/my-namespaceC/0x00000000_0x0FFFFFFF", + Optional.of("broker1:8080")), + Success, Overloaded)); + assertEquals(res, expected); + assertEquals(counter.getLoadAvg(), setupLoadAvg); + assertEquals(counter.getLoadStd(), setupLoadStd); + } + @Test public void testGetAvailableBrokersFailed() { UnloadCounter counter = new UnloadCounter(); From 346bc306cece4756313dd4c0c29a0dff725c1f68 Mon Sep 17 00:00:00 2001 From: Oneby Wang <44369297+oneby-wang@users.noreply.github.com> Date: Wed, 3 Jun 2026 02:47:24 +0800 Subject: [PATCH 024/213] [fix][meta] Fix PulsarZooKeeperClient async addWatch callback retry behavior (#25913) (cherry picked from commit be9f97ac0f833f2dc74dc0f4538e647f7376461f) --- .../metadata/impl/PulsarZooKeeperClient.java | 10 ++-- .../pulsar/metadata/MetadataStoreTest.java | 58 +++++++++++++++++++ 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/PulsarZooKeeperClient.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/PulsarZooKeeperClient.java index 6a995f20e745a..462df69b2ea2a 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/PulsarZooKeeperClient.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/PulsarZooKeeperClient.java @@ -1163,7 +1163,7 @@ public String toString() { } @Override - public void addWatch(String basePath, Watcher watcher, AddWatchMode mode, VoidCallback cb, Object ctx) { + public void addWatch(String basePath, Watcher watcher, AddWatchMode mode, VoidCallback cb, Object context) { final Runnable proc = new ZkRetryRunnable(operationRetryPolicy, rateLimiter, setStats) { final VoidCallback vCb = new VoidCallback() { @@ -1174,7 +1174,7 @@ public void processResult(int rc, String path, Object ctx) { if (allowRetry(worker, rc)) { backOffAndRetry(that, worker.nextRetryWaitTime()); } else { - vCb.processResult(rc, basePath, ctx); + cb.processResult(rc, path, context); } } @@ -1184,15 +1184,15 @@ public void processResult(int rc, String path, Object ctx) { void zkRun() { ZooKeeper zkHandle = zk.get(); if (null == zkHandle) { - PulsarZooKeeperClient.super.addWatch(basePath, watcher, mode, cb, ctx); + PulsarZooKeeperClient.super.addWatch(basePath, watcher, mode, vCb, worker); } else { - zkHandle.addWatch(basePath, watcher, mode, cb, ctx); + zkHandle.addWatch(basePath, watcher, mode, vCb, worker); } } @Override public String toString() { - return String.format("setData (%s, mode = %s)", basePath, mode.name()); + return String.format("addWatch (%s, mode = %s)", basePath, mode.name()); } }; // execute it immediately diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java index 9bd2ddd5e8f22..14810b196929f 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java @@ -19,6 +19,12 @@ package org.apache.pulsar.metadata; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; @@ -40,6 +46,7 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -48,6 +55,7 @@ import lombok.Cleanup; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.zookeeper.BoundExponentialBackoffRetryPolicy; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.GetResult; import org.apache.pulsar.metadata.api.MetadataStore; @@ -62,6 +70,9 @@ import org.apache.pulsar.metadata.impl.PulsarZooKeeperClient; import org.apache.pulsar.metadata.impl.ZKMetadataStore; import org.apache.pulsar.metadata.impl.oxia.OxiaMetadataStore; +import org.apache.zookeeper.AddWatchMode; +import org.apache.zookeeper.AsyncCallback.VoidCallback; +import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; import org.apache.zookeeper.ZooKeeper; @@ -538,6 +549,53 @@ public void testZkLoadConfigFromFile() throws Exception { assertFalse(zooKeeper.getClientConfig().isSaslClientEnabled()); } + @Test + @SuppressWarnings("unchecked") + public void testAsyncAddWatchRetriesWithWrapperCallback() throws Exception { + String path = newKey(); + @Cleanup + PulsarZooKeeperClient zkClient = PulsarZooKeeperClient.newBuilder() + .connectString(zks.getConnectionString()) + .sessionTimeoutMs(3000) + .operationRetryPolicy(new BoundExponentialBackoffRetryPolicy(0, 0, 3)) + .build(); + + ZooKeeper mockZk = mock(ZooKeeper.class); + AtomicInteger attempts = new AtomicInteger(); + doAnswer(invocation -> { + // The wrapper callback should consume this recoverable failure and retry the addWatch operation. + int rc = attempts.incrementAndGet() == 1 + ? KeeperException.Code.CONNECTIONLOSS.intValue() + : KeeperException.Code.OK.intValue(); + String callbackPath = invocation.getArgument(0); + VoidCallback callback = invocation.getArgument(3); + Object callbackContext = invocation.getArgument(4); + callback.processResult(rc, callbackPath, callbackContext); + return null; + }).when(mockZk).addWatch(eq(path), any(Watcher.class), eq(AddWatchMode.PERSISTENT_RECURSIVE), + any(VoidCallback.class), any()); + + // Force the Pulsar wrapper to delegate the async addWatch call to our controlled ZooKeeper instance. + var zooKeeperRef = (AtomicReference) WhiteboxImpl.getInternalState(zkClient, "zk"); + zooKeeperRef.set(mockZk); + + CountDownLatch callbackCalled = new CountDownLatch(1); + AtomicInteger callbackRc = new AtomicInteger(Integer.MIN_VALUE); + zkClient.addWatch(path, event -> { + }, AddWatchMode.PERSISTENT_RECURSIVE, (rc, callbackPath, ctx) -> { + callbackRc.set(rc); + callbackCalled.countDown(); + }, null); + + assertTrue(callbackCalled.await(5, TimeUnit.SECONDS)); + + // The caller should only see the final successful result after the retry, not the first CONNECTIONLOSS. + assertEquals(callbackRc.get(), KeeperException.Code.OK.intValue()); + assertEquals(attempts.get(), 2); + verify(mockZk, times(2)).addWatch(eq(path), any(Watcher.class), eq(AddWatchMode.PERSISTENT_RECURSIVE), + any(VoidCallback.class), any()); + } + @Test public void testOxiaLoadConfigFromFile() throws Exception { final String metadataStoreName = UUID.randomUUID().toString().replaceAll("-", ""); From 6593c77697a17a2959f5d8ec1f4995370b5bede9 Mon Sep 17 00:00:00 2001 From: Oneby Wang <44369297+oneby-wang@users.noreply.github.com> Date: Wed, 3 Jun 2026 05:14:48 +0800 Subject: [PATCH 025/213] [fix][meta] Fix ZooKeeper session reconnect race condition in PulsarZooKeeperClient.clientCreator (#25910) (cherry picked from commit 5627c01b1ef04b8424b781eabc4dea6963faf847) --- .../metadata/impl/PulsarZooKeeperClient.java | 37 ++++++++++++++++--- .../metadata/impl/ZKSessionWatcher.java | 19 +++++++++- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/PulsarZooKeeperClient.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/PulsarZooKeeperClient.java index 462df69b2ea2a..39921b0f89f19 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/PulsarZooKeeperClient.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/PulsarZooKeeperClient.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.Set; import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.RejectedExecutionException; @@ -122,16 +123,42 @@ public ZooKeeper call() throws KeeperException, InterruptedException { log.info("Reconnecting zookeeper {}.", connectString); // close the previous one closeZkHandle(); + + // ZooKeeper can deliver SyncConnected after createZooKeeper() returns but before zk.set(newZk) + // publishes the new instance. Hold these events until the new instance is published, so child + // watchers never observe a new-session event while PulsarZooKeeperClient still points at the + // old handle. + CountDownLatch newZkSetLatch = new CountDownLatch(1); + Watcher forwardEventsWatcher = event -> { + try { + boolean awaited = newZkSetLatch.await(sessionTimeoutMs, TimeUnit.MILLISECONDS); + if (!awaited) { + log.warn("Timed out waiting for ZooKeeper instance to be published before " + + "forwarding event {}", event); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.warn("Interrupted while waiting for ZooKeeper instance to be published, event {}", + event, e); + return; + } + watcherManager.process(event); + }; + ZooKeeper newZk; try { - newZk = createZooKeeper(); + newZk = createZooKeeper(forwardEventsWatcher); } catch (IOException | QuorumPeerConfig.ConfigException e) { log.error("Failed to create zookeeper instance to {} with config path {}", connectString, configPath, e); throw KeeperException.create(KeeperException.Code.CONNECTIONLOSS); } - waitForConnection(); + + // Publish the new instance before releasing the forwarding watcher. waitForConnection() must + // happen after countDown(), since it depends on the forwarded SyncConnected event. zk.set(newZk); + newZkSetLatch.countDown(); + waitForConnection(); log.info("ZooKeeper session {} is created to {}.", Long.toHexString(newZk.getSessionId()), connectString); return newZk; @@ -354,12 +381,12 @@ public void waitForConnection() throws KeeperException, InterruptedException { watcherManager.waitForConnection(); } - protected ZooKeeper createZooKeeper() throws IOException, QuorumPeerConfig.ConfigException { + protected ZooKeeper createZooKeeper(Watcher watcher) throws IOException, QuorumPeerConfig.ConfigException { if (null != configPath) { - return new ZooKeeper(connectString, sessionTimeoutMs, watcherManager, allowReadOnlyMode, + return new ZooKeeper(connectString, sessionTimeoutMs, watcher, allowReadOnlyMode, new ZKClientConfig(configPath)); } - return new ZooKeeper(connectString, sessionTimeoutMs, watcherManager, allowReadOnlyMode); + return new ZooKeeper(connectString, sessionTimeoutMs, watcher, allowReadOnlyMode); } @Override diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java index a840721023080..cc26231a74525 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java @@ -87,6 +87,7 @@ public void close() throws Exception { // in the future. private void checkConnectionStatus() { try { + long checkedSessionId = zk.getSessionId(); CompletableFuture future = new CompletableFuture<>(); zk.exists("/", false, (StatCallback) (rc, path, ctx, stat) -> { switch (KeeperException.Code.get(rc)) { @@ -112,7 +113,7 @@ private void checkConnectionStatus() { zkClientState = Watcher.Event.KeeperState.Disconnected; } - checkState(zkClientState); + checkStateIfSameSession(checkedSessionId, zkClientState); } catch (RejectedExecutionException | InterruptedException e) { task.cancel(true); } catch (Throwable t) { @@ -130,6 +131,22 @@ synchronized void setSessionInvalid() { currentStatus = SessionEvent.SessionLost; } + // PulsarZooKeeperClient publishes the new ZooKeeper instance before forwarding the corresponding session event to + // watcherManager, so zk.set(newZk) happens-before this watcher observes the new-session event. Keep the session-id + // check and state transition in the same synchronized section to prevent stale async probes from racing with that + // event and overwriting the state of the newly established session. + private synchronized void checkStateIfSameSession(long checkedSessionId, + Watcher.Event.KeeperState zkClientState) { + long currentSessionId = zk.getSessionId(); + if (checkedSessionId != currentSessionId) { + log.warn("Ignoring ZooKeeper session state from a stale session. checkedSessionId: {}," + + " currentSessionId: {}, zkClientState: {}", + checkedSessionId, currentSessionId, zkClientState); + return; + } + checkState(zkClientState); + } + private synchronized void checkState(Watcher.Event.KeeperState zkClientState) { switch (zkClientState) { case Expired: From 2820ef4d7f201741f4f343d9b3cba089fc87bc14 Mon Sep 17 00:00:00 2001 From: Oneby Wang <44369297+oneby-wang@users.noreply.github.com> Date: Wed, 3 Jun 2026 03:14:35 +0800 Subject: [PATCH 026/213] [fix][test] Fix flaky SameAuthParamsLookupAutoClusterFailoverTest.testAutoClusterFailover() test (#25892) (cherry picked from commit ed099501840fed113c630f92c5850faff8e118cc) --- .../broker/SameAuthParamsLookupAutoClusterFailoverTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java index febd0c4aad0fa..1314be12c9d0e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java @@ -98,7 +98,6 @@ public void testAutoClusterFailover(boolean enabledTls) throws Exception { .tlsTrustCertsFilePath(CA_CERT_FILE_PATH); } final PulsarClient client = clientBuilder.build(); - failover.initialize(client); final EventLoopGroup executor = WhiteboxImpl.getInternalState(failover, "executor"); final PulsarServiceState[] stateArray = WhiteboxImpl.getInternalState(failover, "pulsarServiceStateArray"); From 1ee0ea79aa8251a5c1610a80f711c5189768e11f Mon Sep 17 00:00:00 2001 From: Pratik Katti <90851204+pratt4@users.noreply.github.com> Date: Wed, 3 Jun 2026 02:45:34 +0530 Subject: [PATCH 027/213] [improve][fn] make built-in functions reload incremental (#25868) (cherry picked from commit d57af8f37725ed16547139978f3a4d83022066c5) --- .../functions/worker/FunctionsManager.java | 30 +++- .../FunctionsManagerReloadFunctionsTest.java | 80 +++++++++ .../utils/functions/FunctionArchive.java | 28 +++ .../utils/functions/FunctionUtils.java | 77 +++++++++ .../functions/ReloadFunctionsResult.java | 29 ++++ .../functions/FunctionUtilsReloadTest.java | 159 ++++++++++++++++++ 6 files changed, 395 insertions(+), 8 deletions(-) create mode 100644 pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/worker/FunctionsManagerReloadFunctionsTest.java create mode 100644 pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/ReloadFunctionsResult.java create mode 100644 pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/functions/FunctionUtilsReloadTest.java diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/FunctionsManager.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/FunctionsManager.java index cdd772495023e..42841d1d695f1 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/FunctionsManager.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/FunctionsManager.java @@ -21,6 +21,7 @@ import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.nio.file.Path; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -30,10 +31,11 @@ import org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory; import org.apache.pulsar.functions.utils.functions.FunctionArchive; import org.apache.pulsar.functions.utils.functions.FunctionUtils; +import org.apache.pulsar.functions.utils.functions.ReloadFunctionsResult; @Slf4j public class FunctionsManager implements AutoCloseable { - private Map functions; + private volatile Map functions; @VisibleForTesting public FunctionsManager() { @@ -62,32 +64,44 @@ public List getFunctionDefinitions() { } public void reloadFunctions(WorkerConfig workerConfig) throws IOException { - Map oldFunctions = functions; - this.functions = createFunctions(workerConfig); - closeFunctions(oldFunctions); + ReloadFunctionsResult reload = FunctionUtils.reloadFunctions( + this.functions, + workerConfig.getFunctionsDirectory(), + workerConfig.getNarExtractionDirectory(), + isEnableClassloading(workerConfig)); + this.functions = reload.functions(); + closeFunctions(reload.functionsToClose()); } private static Map createFunctions(WorkerConfig workerConfig) throws IOException { - boolean enableClassloading = workerConfig.getEnableClassloadingOfBuiltinFiles() - || ThreadRuntimeFactory.class.getName().equals(workerConfig.getFunctionRuntimeFactoryClassName()); + boolean enableClassloading = isEnableClassloading(workerConfig); return FunctionUtils.searchForFunctions(workerConfig.getFunctionsDirectory(), workerConfig.getNarExtractionDirectory(), enableClassloading); } + private static boolean isEnableClassloading(WorkerConfig workerConfig) { + return workerConfig.getEnableClassloadingOfBuiltinFiles() + || ThreadRuntimeFactory.class.getName().equals(workerConfig.getFunctionRuntimeFactoryClassName()); + } + @Override public void close() { closeFunctions(functions); } - private void closeFunctions(Map functionMap) { - functionMap.values().forEach(functionArchive -> { + private void closeFunctions(Collection functions) { + functions.forEach(functionArchive -> { try { functionArchive.close(); } catch (Exception e) { log.warn("Failed to close function archive", e); } }); + } + + private void closeFunctions(Map functionMap) { + closeFunctions(functionMap.values()); functionMap.clear(); } } diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/worker/FunctionsManagerReloadFunctionsTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/worker/FunctionsManagerReloadFunctionsTest.java new file mode 100644 index 0000000000000..0ea2ae2036ad4 --- /dev/null +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/worker/FunctionsManagerReloadFunctionsTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.worker; + +import static org.testng.Assert.assertSame; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import org.apache.pulsar.common.functions.FunctionDefinition; +import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.apache.pulsar.functions.utils.functions.FunctionArchive; +import org.testng.annotations.Test; + +/** + * Tests {@link FunctionsManager#reloadFunctions(WorkerConfig)} for incremental reload behavior, + * ensuring unchanged functions are reused instead of being recreated. + */ +public class FunctionsManagerReloadFunctionsTest { + + private static void writeMinimalNar(Path narPath, FunctionDefinition def) throws IOException { + byte[] yaml = ObjectMapperFactory.getYamlMapper().getObjectMapper().writeValueAsBytes(def); + try (OutputStream os = Files.newOutputStream(narPath); + ZipOutputStream zos = new ZipOutputStream(os)) { + ZipEntry entry = new ZipEntry("META-INF/services/pulsar-io.yaml"); + zos.putNextEntry(entry); + zos.write(yaml); + zos.closeEntry(); + } + } + + private static FunctionDefinition sampleDefinition(String name) { + FunctionDefinition def = new FunctionDefinition(); + def.setName(name); + def.setFunctionClass("org.example.Function"); + return def; + } + + @Test + public void reloadWhenNarUnchangedReusesSameFunctionArchiveInstance() throws Exception { + Path dir = Files.createTempDirectory("mgr-fn-reload-"); + Path nar = dir.resolve("f1.nar"); + writeMinimalNar(nar, sampleDefinition("f-one")); + + WorkerConfig workerConfig = new WorkerConfig(); + workerConfig.setFunctionsDirectory(dir.toString()); + workerConfig.setNarExtractionDirectory(NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR); + workerConfig.setEnableClassloadingOfBuiltinFiles(false); + + try (FunctionsManager manager = new FunctionsManager(workerConfig)) { + FunctionArchive before = manager.getFunction("f-one"); + before.getFunctionPackage(); + + manager.reloadFunctions(workerConfig); + + FunctionArchive after = manager.getFunction("f-one"); + assertSame(after, before); + before.getFunctionPackage(); + } + } +} diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionArchive.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionArchive.java index cfb213f34ed72..70b452eba39cf 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionArchive.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionArchive.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.functions.utils.functions; +import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.file.Path; import org.apache.pulsar.common.functions.FunctionDefinition; import org.apache.pulsar.functions.utils.FunctionFilePackage; @@ -25,6 +27,8 @@ public class FunctionArchive implements AutoCloseable { private final Path archivePath; + /** MD5 hex of archive file contents; empty when {@link #archivePath} is null (test doubles). */ + private final String archiveMd5Hex; private final FunctionDefinition functionDefinition; private final String narExtractionDirectory; private final boolean enableClassloading; @@ -33,16 +37,40 @@ public class FunctionArchive implements AutoCloseable { public FunctionArchive(Path archivePath, FunctionDefinition functionDefinition, String narExtractionDirectory, boolean enableClassloading) { + this(archivePath, functionDefinition, narExtractionDirectory, enableClassloading, null); + } + + /** + * @param precomputedArchiveMd5Hex MD5 hex of {@code archivePath} contents; if null and path is non-null, + * the hash is computed once at construction time. + */ + public FunctionArchive(Path archivePath, FunctionDefinition functionDefinition, String narExtractionDirectory, + boolean enableClassloading, String precomputedArchiveMd5Hex) { this.archivePath = archivePath; this.functionDefinition = functionDefinition; this.narExtractionDirectory = narExtractionDirectory; this.enableClassloading = enableClassloading; + if (archivePath != null) { + try { + this.archiveMd5Hex = precomputedArchiveMd5Hex != null + ? precomputedArchiveMd5Hex + : FunctionUtils.computeArchiveMd5Hex(archivePath); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } else { + this.archiveMd5Hex = ""; + } } public Path getArchivePath() { return archivePath; } + public String getArchiveMd5Hex() { + return archiveMd5Hex; + } + public synchronized ValidatableFunctionPackage getFunctionPackage() { if (closed) { throw new IllegalStateException("FunctionArchive is already closed"); diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java index f4d45edf36301..1425b2985da92 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java @@ -25,12 +25,16 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HexFormat; +import java.util.List; import java.util.Map; import java.util.TreeMap; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.common.functions.FunctionDefinition; +import org.apache.pulsar.common.nar.FileUtils; import org.apache.pulsar.common.nar.NarClassLoader; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.zeroturnaround.zip.ZipUtil; @@ -42,6 +46,17 @@ public class FunctionUtils { private static final String PULSAR_IO_SERVICE_NAME = "pulsar-io.yaml"; + /** + * Computes MD5 digest of a file as lower-case hex (for function archive identity on reload). + */ + public static String computeArchiveMd5Hex(Path path) throws IOException { + return calculateMd5Hex(path.toAbsolutePath().normalize().toFile()); + } + + private static String calculateMd5Hex(File file) throws IOException { + return HexFormat.of().formatHex(FileUtils.calculateMd5sum(file)); + } + /** * Extract the Pulsar Function class from a function or archive. */ @@ -107,4 +122,66 @@ public static Map searchForFunctions(String functionsDi return functions; } + + /** + * Reloads functions from disk against {@code previous}, reusing {@link FunctionArchive} instances when path and + * archive MD5 are unchanged (keeps class loaders open). New or changed archives get new instances. + *

+ * {@link ReloadFunctionsResult#functionsToClose()} lists function archives evicted from the active set (replaced + * or no longer present on disk); the caller must {@link FunctionArchive#close()} each. + * + * @param previous functions from the previous scan (may be empty, never null) + * @param functionsDirectory same semantics as {@link #searchForFunctions} + * @param narExtractionDirectory same semantics as {@link #searchForFunctions} + * @param enableClassloading same semantics as {@link #searchForFunctions} + * @return new map keyed by function name (reused values are identical instances from {@code previous}) and + * functions the caller should close + */ + public static ReloadFunctionsResult reloadFunctions( + Map previous, + String functionsDirectory, + String narExtractionDirectory, + boolean enableClassloading) throws IOException { + + TreeMap remaining = new TreeMap<>(previous); + TreeMap next = new TreeMap<>(); + List toClose = new ArrayList<>(); + + Path dir = Paths.get(functionsDirectory).toAbsolutePath().normalize(); + if (!dir.toFile().exists()) { + toClose.addAll(remaining.values()); + return new ReloadFunctionsResult(next, toClose); + } + + try (DirectoryStream stream = Files.newDirectoryStream(dir, "*.nar")) { + for (Path archive : stream) { + try { + FunctionDefinition funcDef = FunctionUtils.getFunctionDefinition(archive.toFile()); + if (!StringUtils.isEmpty(funcDef.getFunctionClass())) { + String name = funcDef.getName(); + String md5Hex = computeArchiveMd5Hex(archive); + FunctionArchive prev = remaining.remove(name); + if (prev != null + && prev.getArchivePath() != null + && archive.equals(prev.getArchivePath()) + && md5Hex.equals(prev.getArchiveMd5Hex())) { + next.put(name, prev); + } else { + if (prev != null) { + log.info("Reloading changed function {} from {} (previous archive {})", + name, archive, prev.getArchivePath()); + toClose.add(prev); + } + next.put(name, new FunctionArchive(archive, funcDef, narExtractionDirectory, + enableClassloading, md5Hex)); + } + } + } catch (Throwable t) { + log.warn("Failed to load function from {}", archive, t); + } + } + } + toClose.addAll(remaining.values()); + return new ReloadFunctionsResult(next, toClose); + } } diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/ReloadFunctionsResult.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/ReloadFunctionsResult.java new file mode 100644 index 0000000000000..2eb8c73644512 --- /dev/null +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/ReloadFunctionsResult.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.utils.functions; + +import java.util.List; +import java.util.Map; + +/** + * Result of {@link FunctionUtils#reloadFunctions}: the new function map and function archives evicted from the + * active set that the caller must close. + */ +public record ReloadFunctionsResult(Map functions, List functionsToClose) { +} diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/functions/FunctionUtilsReloadTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/functions/FunctionUtilsReloadTest.java new file mode 100644 index 0000000000000..0dec5f1c613e5 --- /dev/null +++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/functions/FunctionUtilsReloadTest.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.utils.functions; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotSame; +import static org.testng.Assert.assertSame; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import org.apache.pulsar.common.functions.FunctionDefinition; +import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.testng.annotations.Test; + +@Test +public class FunctionUtilsReloadTest { + + private static void closeEvicted(ReloadFunctionsResult reload) throws Exception { + for (FunctionArchive functionArchive : reload.functionsToClose()) { + functionArchive.close(); + } + } + + private static void writeMinimalNar(Path narPath, FunctionDefinition def) throws IOException { + byte[] yaml = ObjectMapperFactory.getYamlMapper().getObjectMapper().writeValueAsBytes(def); + try (OutputStream os = Files.newOutputStream(narPath); + ZipOutputStream zos = new ZipOutputStream(os)) { + ZipEntry entry = new ZipEntry("META-INF/services/pulsar-io.yaml"); + zos.putNextEntry(entry); + zos.write(yaml); + zos.closeEntry(); + } + } + + private static FunctionDefinition sampleDefinition(String name) { + FunctionDefinition def = new FunctionDefinition(); + def.setName(name); + def.setFunctionClass("org.example.Function"); + return def; + } + + /** + * Historical {@code FunctionsManager} reload replaced the whole map and closed every prior + * {@link FunctionArchive}, even when NAR files were unchanged. A caller keeping a reference to the + * pre-reload archive would then hit {@link IllegalStateException} on lazy use. + *

+ * Incremental reload must evict nothing, reuse the same instance, and leave that instance usable + * after the caller closes only {@link ReloadFunctionsResult#functionsToClose()}. + */ + @Test + public void reloadUnchangedNarEvictsNothingAndKeepsSameFunctionArchiveUsable() throws Exception { + Path dir = Files.createTempDirectory("fn-reload-"); + Path nar = dir.resolve("f1.nar"); + writeMinimalNar(nar, sampleDefinition("f-one")); + + Map first = + FunctionUtils.searchForFunctions(dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); + FunctionArchive functionArchive = first.get("f-one"); + functionArchive.getFunctionPackage(); + + ReloadFunctionsResult reload = FunctionUtils.reloadFunctions( + first, dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); + assertTrue(reload.functionsToClose().isEmpty()); + closeEvicted(reload); + Map second = reload.functions(); + + assertSame(second.get("f-one"), functionArchive); + functionArchive.getFunctionPackage(); + } + + @Test + public void reloadReopensFunctionArchiveWhenNarContentChanges() throws Exception { + Path dir = Files.createTempDirectory("fn-reload-"); + Path nar = dir.resolve("f1.nar"); + writeMinimalNar(nar, sampleDefinition("f-one")); + + Map first = + FunctionUtils.searchForFunctions(dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); + FunctionArchive before = first.get("f-one"); + + FunctionDefinition updated = sampleDefinition("f-one"); + updated.setDescription("changed"); + writeMinimalNar(nar, updated); + + ReloadFunctionsResult reload = FunctionUtils.reloadFunctions( + first, dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); + closeEvicted(reload); + Map second = reload.functions(); + + assertNotSame(second.get("f-one"), before); + assertThrows(IllegalStateException.class, before::getFunctionPackage); + } + + @Test + public void reloadClosesFunctionArchivesRemovedFromDirectory() throws Exception { + Path dir = Files.createTempDirectory("fn-reload-"); + Path nar1 = dir.resolve("a.nar"); + Path nar2 = dir.resolve("b.nar"); + writeMinimalNar(nar1, sampleDefinition("fn-a")); + writeMinimalNar(nar2, sampleDefinition("fn-b")); + + Map first = + FunctionUtils.searchForFunctions(dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); + FunctionArchive removed = first.get("fn-b"); + Files.delete(nar2); + + ReloadFunctionsResult reload = FunctionUtils.reloadFunctions( + first, dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); + closeEvicted(reload); + Map second = reload.functions(); + + assertEquals(second.size(), 1); + assertSame(second.get("fn-a"), first.get("fn-a")); + assertThrows(IllegalStateException.class, removed::getFunctionPackage); + } + + @Test + public void reloadClosesAllFunctionArchivesWhenDirectoryIsMissing() throws Exception { + Path dir = Files.createTempDirectory("fn-reload-"); + Path nar = dir.resolve("f1.nar"); + writeMinimalNar(nar, sampleDefinition("f-one")); + + Map first = + FunctionUtils.searchForFunctions(dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); + FunctionArchive removed = first.get("f-one"); + Files.delete(nar); + Files.delete(dir); + + ReloadFunctionsResult reload = FunctionUtils.reloadFunctions( + first, dir.toString(), NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, false); + closeEvicted(reload); + + assertTrue(reload.functions().isEmpty()); + assertThrows(IllegalStateException.class, removed::getFunctionPackage); + } +} From f053c97dcb28fb10291bf0a48592114b7db3652a Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 3 Jun 2026 05:06:59 +0300 Subject: [PATCH 028/213] [fix][sec] Upgrade Netty to 4.1.135.Final to address several CVEs (#25918) (cherry picked from commit df953e9d7e95a052a44a906229750824338e1711) --- .../server/src/assemble/LICENSE.bin.txt | 40 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 38 +++++++++--------- pom.xml | 2 +- 3 files changed, 40 insertions(+), 40 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 39aa9ada72ae3..b26c3534ec7a8 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -293,26 +293,26 @@ The Apache Software License, Version 2.0 - org.apache.commons-commons-lang3-3.19.0.jar - org.apache.commons-commons-text-1.14.0.jar * Netty - - io.netty-netty-buffer-4.1.134.Final.jar - - io.netty-netty-codec-4.1.134.Final.jar - - io.netty-netty-codec-dns-4.1.134.Final.jar - - io.netty-netty-codec-http-4.1.134.Final.jar - - io.netty-netty-codec-http2-4.1.134.Final.jar - - io.netty-netty-codec-socks-4.1.134.Final.jar - - io.netty-netty-codec-haproxy-4.1.134.Final.jar - - io.netty-netty-common-4.1.134.Final.jar - - io.netty-netty-handler-4.1.134.Final.jar - - io.netty-netty-handler-proxy-4.1.134.Final.jar - - io.netty-netty-resolver-4.1.134.Final.jar - - io.netty-netty-resolver-dns-4.1.134.Final.jar - - io.netty-netty-resolver-dns-classes-macos-4.1.134.Final.jar - - io.netty-netty-resolver-dns-native-macos-4.1.134.Final-osx-aarch_64.jar - - io.netty-netty-resolver-dns-native-macos-4.1.134.Final-osx-x86_64.jar - - io.netty-netty-transport-4.1.134.Final.jar - - io.netty-netty-transport-classes-epoll-4.1.134.Final.jar - - io.netty-netty-transport-native-epoll-4.1.134.Final-linux-aarch_64.jar - - io.netty-netty-transport-native-epoll-4.1.134.Final-linux-x86_64.jar - - io.netty-netty-transport-native-unix-common-4.1.134.Final.jar + - io.netty-netty-buffer-4.1.135.Final.jar + - io.netty-netty-codec-4.1.135.Final.jar + - io.netty-netty-codec-dns-4.1.135.Final.jar + - io.netty-netty-codec-http-4.1.135.Final.jar + - io.netty-netty-codec-http2-4.1.135.Final.jar + - io.netty-netty-codec-socks-4.1.135.Final.jar + - io.netty-netty-codec-haproxy-4.1.135.Final.jar + - io.netty-netty-common-4.1.135.Final.jar + - io.netty-netty-handler-4.1.135.Final.jar + - io.netty-netty-handler-proxy-4.1.135.Final.jar + - io.netty-netty-resolver-4.1.135.Final.jar + - io.netty-netty-resolver-dns-4.1.135.Final.jar + - io.netty-netty-resolver-dns-classes-macos-4.1.135.Final.jar + - io.netty-netty-resolver-dns-native-macos-4.1.135.Final-osx-aarch_64.jar + - io.netty-netty-resolver-dns-native-macos-4.1.135.Final-osx-x86_64.jar + - io.netty-netty-transport-4.1.135.Final.jar + - io.netty-netty-transport-classes-epoll-4.1.135.Final.jar + - io.netty-netty-transport-native-epoll-4.1.135.Final-linux-aarch_64.jar + - io.netty-netty-transport-native-epoll-4.1.135.Final-linux-x86_64.jar + - io.netty-netty-transport-native-unix-common-4.1.135.Final.jar - io.netty-netty-tcnative-boringssl-static-2.0.77.Final.jar - io.netty-netty-tcnative-boringssl-static-2.0.77.Final-linux-aarch_64.jar - io.netty-netty-tcnative-boringssl-static-2.0.77.Final-linux-x86_64.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 58c6e330f20b5..a51b2161e9b3a 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -345,22 +345,22 @@ The Apache Software License, Version 2.0 - commons-text-1.14.0.jar - commons-compress-1.28.0.jar * Netty - - netty-buffer-4.1.134.Final.jar - - netty-codec-4.1.134.Final.jar - - netty-codec-dns-4.1.134.Final.jar - - netty-codec-http-4.1.134.Final.jar - - netty-codec-socks-4.1.134.Final.jar - - netty-codec-haproxy-4.1.134.Final.jar - - netty-common-4.1.134.Final.jar - - netty-handler-4.1.134.Final.jar - - netty-handler-proxy-4.1.134.Final.jar - - netty-resolver-4.1.134.Final.jar - - netty-resolver-dns-4.1.134.Final.jar - - netty-transport-4.1.134.Final.jar - - netty-transport-classes-epoll-4.1.134.Final.jar - - netty-transport-native-epoll-4.1.134.Final-linux-aarch_64.jar - - netty-transport-native-epoll-4.1.134.Final-linux-x86_64.jar - - netty-transport-native-unix-common-4.1.134.Final.jar + - netty-buffer-4.1.135.Final.jar + - netty-codec-4.1.135.Final.jar + - netty-codec-dns-4.1.135.Final.jar + - netty-codec-http-4.1.135.Final.jar + - netty-codec-socks-4.1.135.Final.jar + - netty-codec-haproxy-4.1.135.Final.jar + - netty-common-4.1.135.Final.jar + - netty-handler-4.1.135.Final.jar + - netty-handler-proxy-4.1.135.Final.jar + - netty-resolver-4.1.135.Final.jar + - netty-resolver-dns-4.1.135.Final.jar + - netty-transport-4.1.135.Final.jar + - netty-transport-classes-epoll-4.1.135.Final.jar + - netty-transport-native-epoll-4.1.135.Final-linux-aarch_64.jar + - netty-transport-native-epoll-4.1.135.Final-linux-x86_64.jar + - netty-transport-native-unix-common-4.1.135.Final.jar - netty-tcnative-boringssl-static-2.0.77.Final.jar - netty-tcnative-boringssl-static-2.0.77.Final-linux-aarch_64.jar - netty-tcnative-boringssl-static-2.0.77.Final-linux-x86_64.jar @@ -371,9 +371,9 @@ The Apache Software License, Version 2.0 - netty-incubator-transport-classes-io_uring-0.0.26.Final.jar - netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar - netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar - - netty-resolver-dns-classes-macos-4.1.134.Final.jar - - netty-resolver-dns-native-macos-4.1.134.Final-osx-aarch_64.jar - - netty-resolver-dns-native-macos-4.1.134.Final-osx-x86_64.jar + - netty-resolver-dns-classes-macos-4.1.135.Final.jar + - netty-resolver-dns-native-macos-4.1.135.Final-osx-aarch_64.jar + - netty-resolver-dns-native-macos-4.1.135.Final-osx-x86_64.jar * Prometheus client - simpleclient-0.16.0.jar - simpleclient_log4j2-0.16.0.jar diff --git a/pom.xml b/pom.xml index 3766bdcc06dc4..46faf2109a7b0 100644 --- a/pom.xml +++ b/pom.xml @@ -187,7 +187,7 @@ flexible messaging model and an intuitive client API. 1.1.10.8 4.1.12.1 5.7.1 - 4.1.134.Final + 4.1.135.Final 0.0.26.Final 12.1.9 From baa63c6d7106b86860b365557db2842afb20619a Mon Sep 17 00:00:00 2001 From: Dream95 <864197662@qq.com> Date: Wed, 3 Jun 2026 16:52:06 +0800 Subject: [PATCH 029/213] [fix][client] Clean up unacked messages when unsubscribing a topic with ack timeout backoff (#25916) Signed-off-by: Dream95 (cherry picked from commit 756c03df2d3ef17ce60ee124bbef81be03fc0191) --- .../client/impl/MultiTopicsConsumerImpl.java | 2 + .../UnAckedTopicMessageRedeliveryTracker.java | 5 +- ...ckedTopicMessageRedeliveryTrackerTest.java | 79 +++++++++++++++++++ 3 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTrackerTest.java diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index f5f24cb5b89d5..560d62851aa89 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -1304,6 +1304,8 @@ public CompletableFuture unsubscribeAsync(String topicName) { removeTopic(topicName); if (unAckedMessageTracker instanceof UnAckedTopicMessageTracker) { ((UnAckedTopicMessageTracker) unAckedMessageTracker).removeTopicMessages(topicName); + } else if (unAckedMessageTracker instanceof UnAckedTopicMessageRedeliveryTracker) { + ((UnAckedTopicMessageRedeliveryTracker) unAckedMessageTracker).removeTopicMessages(topicName); } unsubscribeFuture.complete(null); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTracker.java index 823dd4ad5f488..393557cd89adb 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTracker.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTracker.java @@ -44,7 +44,6 @@ public int removeTopicMessages(String topicName) { MessageId messageId = messageIdWrapper.getMessageId(); if (messageId instanceof TopicMessageId && ((TopicMessageId) messageId).getOwnerTopic().contains(topicName)) { - HashSet exist = redeliveryMessageIdPartitionMap.get(messageIdWrapper); entry.getValue().remove(messageIdWrapper); iterator.remove(); messageIdWrapper.recycle(); @@ -53,11 +52,11 @@ public int removeTopicMessages(String topicName) { } Iterator iteratorAckTimeOut = ackTimeoutMessages.keySet().iterator(); - while (iterator.hasNext()) { + while (iteratorAckTimeOut.hasNext()) { MessageId messageId = iteratorAckTimeOut.next(); if (messageId instanceof TopicMessageId && ((TopicMessageId) messageId).getOwnerTopic().contains(topicName)) { - iterator.remove(); + iteratorAckTimeOut.remove(); removed++; } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTrackerTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTrackerTest.java new file mode 100644 index 0000000000000..9fdab39145bd8 --- /dev/null +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTrackerTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import io.netty.util.HashedWheelTimer; +import io.netty.util.Timer; +import io.netty.util.concurrent.DefaultThreadFactory; +import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; +import org.apache.pulsar.client.impl.metrics.InstrumentProvider; +import org.testng.annotations.Test; + +public class UnAckedTopicMessageRedeliveryTrackerTest { + + @Test + @SuppressWarnings("unchecked") + public void testRemoveTopicMessages() { + PulsarClientImpl client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.instrumentProvider()).thenReturn(InstrumentProvider.NOOP); + when(client.getCnxPool()).thenReturn(connectionPool); + @Cleanup("stop") + Timer timer = new HashedWheelTimer( + new DefaultThreadFactory("pulsar-timer", Thread.currentThread().isDaemon()), + 1, TimeUnit.MILLISECONDS); + when(client.timer()).thenReturn(timer); + + ConsumerBase consumer = mock(ConsumerBase.class); + doNothing().when(consumer).onAckTimeoutSend(any()); + doNothing().when(consumer).redeliverUnacknowledgedMessages(any()); + + ConsumerConfigurationData conf = new ConsumerConfigurationData<>(); + conf.setAckTimeoutMillis(1_000_000); + conf.setTickDurationMillis(100_000); + conf.setAckTimeoutRedeliveryBackoff(MultiplierRedeliveryBackoff.builder().build()); + + UnAckedTopicMessageRedeliveryTracker tracker = + new UnAckedTopicMessageRedeliveryTracker(client, consumer, conf); + + String ownerTopic = "persistent://public/default/my-topic-partition-0"; + TopicMessageIdImpl msgInPartition = + new TopicMessageIdImpl(ownerTopic, new MessageIdImpl(1L, 0L, -1)); + TopicMessageIdImpl msgInAckTimeout = + new TopicMessageIdImpl(ownerTopic, new MessageIdImpl(2L, 0L, -1)); + + assertTrue(tracker.add(msgInPartition)); + tracker.ackTimeoutMessages.put(msgInAckTimeout, System.currentTimeMillis() + 1_000_000L); + assertEquals(tracker.size(), 2); + + assertEquals(tracker.removeTopicMessages("my-topic"), 2); + assertTrue(tracker.isEmpty()); + + tracker.close(); + } + +} From 41a8708ade281a4ee151b86b5db00c5145863fd3 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 3 Jun 2026 21:49:35 +0300 Subject: [PATCH 030/213] [improve][proxy][branch-4.0] Restore AdminProxyHandler changes which were accidentially reverted in Jetty 12 upgrade --- .../proxy/server/AdminProxyHandler.java | 90 +++---------------- 1 file changed, 11 insertions(+), 79 deletions(-) diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java index 23ebed3420caf..7992cd20d11af 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java @@ -26,12 +26,10 @@ import java.util.Arrays; import java.util.HashSet; import java.util.Set; -import java.util.concurrent.Executor; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import javax.net.ssl.SSLContext; -import javax.servlet.ServletConfig; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -44,20 +42,17 @@ import org.apache.pulsar.common.util.PulsarSslConfiguration; import org.apache.pulsar.common.util.PulsarSslFactory; import org.apache.pulsar.policies.data.loadbalancer.ServiceLookupData; -import org.eclipse.jetty.client.ContinueProtocolHandler; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.ProtocolHandlers; import org.eclipse.jetty.client.RedirectProtocolHandler; import org.eclipse.jetty.client.Request; import org.eclipse.jetty.client.transport.HttpClientTransportOverHTTP; import org.eclipse.jetty.ee8.proxy.ProxyServlet; -import org.eclipse.jetty.http.HttpCookieStore; import org.eclipse.jetty.http.HttpField; import org.eclipse.jetty.http.HttpHeader; import org.eclipse.jetty.io.Content; import org.eclipse.jetty.util.BufferUtil; import org.eclipse.jetty.util.ssl.SslContextFactory; -import org.eclipse.jetty.util.thread.QueuedThreadPool; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -112,89 +107,26 @@ class AdminProxyHandler extends ProxyServlet { TimeUnit.SECONDS); } } - super.setTimeout(config.getHttpProxyTimeout()); } @Override protected HttpClient createHttpClient() throws ServletException { - ServletConfig config = getServletConfig(); - - HttpClient client = newHttpClient(); - - client.setFollowRedirects(true); - - // Must not store cookies, otherwise cookies of different clients will mix. - client.setHttpCookieStore(new HttpCookieStore.Empty()); - - Executor executor; - String value = config.getInitParameter("maxThreads"); - if (value == null || "-".equals(value)) { - executor = (Executor) getServletContext().getAttribute("org.eclipse.jetty.server.Executor"); - if (executor == null) { - throw new IllegalStateException("No server executor for proxy"); - } - } else { - QueuedThreadPool qtp = new QueuedThreadPool(Integer.parseInt(value)); - String servletName = config.getServletName(); - int dot = servletName.lastIndexOf('.'); - if (dot >= 0) { - servletName = servletName.substring(dot + 1); - } - qtp.setName(servletName); - executor = qtp; - } - - client.setExecutor(executor); - - value = config.getInitParameter("maxConnections"); - if (value == null) { - value = "256"; - } - client.setMaxConnectionsPerDestination(Integer.parseInt(value)); - - value = config.getInitParameter("idleTimeout"); - if (value == null) { - value = "30000"; - } - client.setIdleTimeout(Long.parseLong(value)); - - value = config.getInitParameter(INIT_PARAM_REQUEST_BUFFER_SIZE); - if (value != null) { - client.setRequestBufferSize(Integer.parseInt(value)); - } - - value = config.getInitParameter("responseBufferSize"); - if (value != null){ - client.setResponseBufferSize(Integer.parseInt(value)); - } - - try { - client.start(); - - // Content must not be decoded, otherwise the client gets confused. - // Allow encoded content, such as "Content-Encoding: gzip", to pass through without decoding it. - client.getContentDecoderFactories().clear(); + HttpClient httpClient = super.createHttpClient(); + customizeHttpClient(httpClient); + return httpClient; + } - // Pass traffic to the client, only intercept what's necessary. - ProtocolHandlers protocolHandlers = client.getProtocolHandlers(); - protocolHandlers.clear(); - protocolHandlers.put(new RedirectProtocolHandler(client)); - protocolHandlers.put(new ProxyContinueProtocolHandler()); + protected void customizeHttpClient(HttpClient httpClient) { + httpClient.setFollowRedirects(true); - return client; - } catch (Exception x) { - throw new ServletException(x); + ProtocolHandlers protocolHandlers = httpClient.getProtocolHandlers(); + if (protocolHandlers != null) { + protocolHandlers.put(new RedirectProtocolHandler(httpClient)); } - } - class ProxyContinueProtocolHandler extends ContinueProtocolHandler { + httpClient.setIdleTimeout(config.getHttpProxyIdleTimeout()); - @Override - protected Runnable onContinue(Request request) { - HttpServletRequest clientRequest = - (HttpServletRequest) request.getAttributes().get(CLIENT_REQUEST_ATTRIBUTE); - return AdminProxyHandler.this.onContinue(clientRequest, request); - } + setTimeout(config.getHttpProxyTimeout()); } // This class allows the request body to be replayed, the default implementation From 016d10174fb8012cce151800743801f5c4e5b0b9 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 3 Jun 2026 11:54:30 +0300 Subject: [PATCH 031/213] [fix][proxy] Avoid intermittent 502 when admin proxy follows a broker redirect for a request with a body (#25919) (cherry picked from commit 2acee322ef8ace78759677b997a57be454ed2eca) --- .../proxy/server/AdminProxyHandler.java | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java index 7992cd20d11af..ba49714b89bd6 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java @@ -46,6 +46,7 @@ import org.eclipse.jetty.client.ProtocolHandlers; import org.eclipse.jetty.client.RedirectProtocolHandler; import org.eclipse.jetty.client.Request; +import org.eclipse.jetty.client.Response; import org.eclipse.jetty.client.transport.HttpClientTransportOverHTTP; import org.eclipse.jetty.ee8.proxy.ProxyServlet; import org.eclipse.jetty.http.HttpField; @@ -121,7 +122,7 @@ protected void customizeHttpClient(HttpClient httpClient) { ProtocolHandlers protocolHandlers = httpClient.getProtocolHandlers(); if (protocolHandlers != null) { - protocolHandlers.put(new RedirectProtocolHandler(httpClient)); + protocolHandlers.put(new NonAbortingRedirectProtocolHandler(httpClient)); } httpClient.setIdleTimeout(config.getHttpProxyIdleTimeout()); @@ -129,6 +130,34 @@ protected void customizeHttpClient(HttpClient httpClient) { setTimeout(config.getHttpProxyTimeout()); } + /** + * A {@link RedirectProtocolHandler} that does not abort the in-flight request when a redirect + * response is received. + * + *

Jetty's default {@link RedirectProtocolHandler#onSuccess(Response)} aborts a request that + * still has a body to send when a redirect status is received, raising + * {@code HttpRequestException: "Aborting request after receiving a NNN response"}. When a broker + * returns a 307 (to redirect an admin request to the bundle-owner broker) before the proxy has + * finished streaming the request body, that abort can race ahead of the redirect continuation in + * {@link RedirectProtocolHandler#onComplete} and surface to the proxy as a spurious HTTP 502 Bad + * Gateway. The redirect itself is driven by {@code onComplete} from the response (its status and + * {@code Location} header) and does not depend on the abort, so skipping it lets the redirect + * always be followed on a fresh request (with the body replayed by + * {@link ReplayableProxyContentProvider}), which is the behavior this proxy needs. + */ + static class NonAbortingRedirectProtocolHandler extends RedirectProtocolHandler { + NonAbortingRedirectProtocolHandler(HttpClient client) { + super(client); + } + + @Override + public void onSuccess(Response response) { + // Intentionally do NOT abort the request here. The redirect is followed in onComplete(); + // aborting the in-flight request only races a 502 to the proxy when the broker returns a + // redirect before the request body has finished sending. + } + } + // This class allows the request body to be replayed, the default implementation // does not protected class ReplayableProxyContentProvider extends ProxyInputStreamRequestContent { From 309fcf881672adaa446d45801b781e5c9f1dcbaf Mon Sep 17 00:00:00 2001 From: Oneby Wang <44369297+oneby-wang@users.noreply.github.com> Date: Thu, 4 Jun 2026 01:44:38 +0800 Subject: [PATCH 032/213] [fix][client] Prevent duplicate ServiceUrlProvider initialization (#25899) (cherry picked from commit 882946c59d37b2a4d7e474bfa3a7d8da4d7fa59d) --- ...thParamsLookupAutoClusterFailoverTest.java | 14 +++++++++++++ .../pulsar/client/api/ServiceUrlProvider.java | 11 +++++++++- .../client/impl/AutoClusterFailover.java | 15 +++++++++++-- .../impl/ControlledClusterFailover.java | 15 +++++++++++-- ...meAuthParamsLookupAutoClusterFailover.java | 16 +++++++++++--- .../client/impl/AutoClusterFailoverTest.java | 21 +++++++++++++++++++ .../impl/ControlledClusterFailoverTest.java | 17 +++++++++++++++ 7 files changed, 101 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java index 1314be12c9d0e..a2e587fef395f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java @@ -172,6 +172,20 @@ public void testAutoClusterFailover(boolean enabledTls) throws Exception { dummyServer.close(); } + @Test + public void testInitializeCanOnlyBeCalledOnce() throws Exception { + setup(); + final SameAuthParamsLookupAutoClusterFailover failover = SameAuthParamsLookupAutoClusterFailover.builder() + .pulsarServiceUrlArray(new String[]{pulsar1.getBrokerServiceUrl()}) + .checkHealthyIntervalMs(1000) + .build(); + + try (PulsarClient client = PulsarClient.builder().serviceUrlProvider(failover).build()) { + Throwable error = Assert.expectThrows(IllegalStateException.class, () -> failover.initialize(client)); + Assert.assertEquals(error.getMessage(), "ServiceUrlProvider has already been initialized"); + } + } + @Override protected void cleanupPulsarResources() { // Nothing to do. diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ServiceUrlProvider.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ServiceUrlProvider.java index e8b513b103f65..f95f650cbb731 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ServiceUrlProvider.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ServiceUrlProvider.java @@ -27,6 +27,11 @@ *

This allows applications to retrieve the service URL from an external configuration provider and, * more importantly, to force the Pulsar client to reconnect if the service URL has been changed. * + *

Each provider instance is tied to the lifecycle of one {@link PulsarClient} instance. The client + * initializes the provider when the client is created and closes the provider when the owning client is + * closed. Applications that create multiple Pulsar clients should create a separate provider instance + * for each client instead of sharing one provider. + * *

It can be passed with {@link ClientBuilder#serviceUrlProvider(ServiceUrlProvider)} */ @InterfaceAudience.Public @@ -39,6 +44,9 @@ public interface ServiceUrlProvider extends AutoCloseable { *

This can be used by the provider to force the Pulsar client to reconnect whenever the service url might have * changed. See {@link PulsarClient#updateServiceUrl(String)}. * + *

This method is invoked by the Pulsar client and is expected to be called once for a provider + * instance. Implementations may reject repeated initialization. + * * @param client * created pulsar client. */ @@ -52,7 +60,8 @@ public interface ServiceUrlProvider extends AutoCloseable { String getServiceUrl(); /** - * Close the resource that the provider allocated. + * Close the resource that the provider allocated. The owning Pulsar client invokes this method when + * it is closed. * */ @Override diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AutoClusterFailover.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AutoClusterFailover.java index 844d1e2d25349..5657d5067c094 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AutoClusterFailover.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AutoClusterFailover.java @@ -39,6 +39,14 @@ import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.client.util.ExecutorProvider; +/** + * A service URL provider that automatically fails over from the primary Pulsar service URL to one of + * the secondary service URLs and switches back after the primary service recovers. + * + *

Each instance is tied to the lifecycle of one {@link PulsarClient}. Once initialized by a + * Pulsar client, it must not be reused by another client. Create a new provider instance for each + * Pulsar client. + */ @Slf4j @Data public class AutoClusterFailover implements ServiceUrlProvider { @@ -84,7 +92,10 @@ private AutoClusterFailover(AutoClusterFailoverBuilderImpl builder) { } @Override - public void initialize(PulsarClient client) { + public synchronized void initialize(PulsarClient client) { + if (this.pulsarClient != null) { + throw new IllegalStateException("ServiceUrlProvider has already been initialized"); + } this.pulsarClient = (PulsarClientImpl) client; ClientConfigurationData config = pulsarClient.getConfiguration(); if (config != null) { @@ -120,7 +131,7 @@ public String getServiceUrl() { } @Override - public void close() { + public synchronized void close() { this.executor.shutdown(); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ControlledClusterFailover.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ControlledClusterFailover.java index 1d4740b847240..db3d7939d3b54 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ControlledClusterFailover.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ControlledClusterFailover.java @@ -55,6 +55,14 @@ import org.asynchttpclient.channel.DefaultKeepAliveStrategy; import org.jspecify.annotations.Nullable; +/** + * A service URL provider that fetches controlled failover configuration from an external HTTP service + * and updates the Pulsar client when the returned configuration changes. + * + *

Each instance is tied to the lifecycle of one {@link PulsarClient}. Once initialized by a + * Pulsar client, it must not be reused by another client. Create a new provider instance for each + * Pulsar client. + */ @Slf4j public class ControlledClusterFailover implements ServiceUrlProvider { private static final int DEFAULT_CONNECT_TIMEOUT_IN_SECONDS = 10; @@ -107,7 +115,10 @@ public boolean keepAlive(InetSocketAddress remoteAddress, Request ahcRequest, } @Override - public void initialize(PulsarClient client) { + public synchronized void initialize(PulsarClient client) { + if (this.pulsarClient != null) { + throw new IllegalStateException("ServiceUrlProvider has already been initialized"); + } this.pulsarClient = (PulsarClientImpl) client; // Initialize currentControlledConfiguration from client's current configuration @@ -210,7 +221,7 @@ public String getServiceUrl() { } @Override - public void close() { + public synchronized void close() { this.executor.shutdown(); if (httpClient != null) { try { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java index 448dd95c4fdd9..69c74bd28d92a 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java @@ -37,6 +37,14 @@ import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.netty.EventLoopUtil; +/** + * A service URL provider that probes multiple Pulsar service URLs with the same authentication + * parameters and fails over according to service health. + * + *

Each instance is tied to the lifecycle of one {@link PulsarClient}. Once initialized by a + * Pulsar client, it must not be reused by another client. Create a new provider instance for each + * Pulsar client. + */ @Slf4j @SuppressFBWarnings(value = {"EI_EXPOSE_REP2"}) public class SameAuthParamsLookupAutoClusterFailover implements ServiceUrlProvider { @@ -65,7 +73,10 @@ public class SameAuthParamsLookupAutoClusterFailover implements ServiceUrlProvid private SameAuthParamsLookupAutoClusterFailover() {} @Override - public void initialize(PulsarClient client) { + public synchronized void initialize(PulsarClient client) { + if (this.pulsarClient != null) { + throw new IllegalStateException("ServiceUrlProvider has already been initialized"); + } this.currentPulsarServiceIndex = 0; this.pulsarClient = (PulsarClientImpl) client; this.executor = EventLoopUtil.newEventLoopGroup(1, false, @@ -111,7 +122,7 @@ public String getServiceUrl() { } @Override - public void close() throws Exception { + public synchronized void close() throws Exception { if (closed) { return; } @@ -371,4 +382,3 @@ public SameAuthParamsLookupAutoClusterFailover build() { } } } - diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AutoClusterFailoverTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AutoClusterFailoverTest.java index b275ffb6012ca..00688db800fe3 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AutoClusterFailoverTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AutoClusterFailoverTest.java @@ -31,10 +31,12 @@ import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.AuthenticationFactory; +import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.ServiceUrlProvider; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.awaitility.Awaitility; import org.mockito.Mockito; +import org.testng.Assert; import org.testng.annotations.Test; @Test(groups = "broker-impl") @@ -148,6 +150,25 @@ public void testInitialize() throws Exception { } } + @Test + public void testInitializeCanOnlyBeCalledOnce() throws Exception { + String primary = "pulsar://localhost:6650"; + String secondary = "pulsar://localhost:6651"; + + ServiceUrlProvider provider = AutoClusterFailover.builder() + .primary(primary) + .secondary(Collections.singletonList(secondary)) + .failoverDelay(1, TimeUnit.SECONDS) + .switchBackDelay(1, TimeUnit.SECONDS) + .checkInterval(30, TimeUnit.SECONDS) + .build(); + + try (PulsarClient client = PulsarClient.builder().serviceUrlProvider(provider).build()) { + Throwable error = Assert.expectThrows(IllegalStateException.class, () -> provider.initialize(client)); + assertEquals(error.getMessage(), "ServiceUrlProvider has already been initialized"); + } + } + @Test public void testAutoClusterFailoverSwitchWithoutAuthentication() throws Exception { String primary = "pulsar://localhost:6650"; diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ControlledClusterFailoverTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ControlledClusterFailoverTest.java index 86b2fa7cb4f9b..cc47841ce0a11 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ControlledClusterFailoverTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ControlledClusterFailoverTest.java @@ -25,6 +25,7 @@ import java.util.concurrent.TimeUnit; import lombok.Cleanup; import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.ServiceUrlProvider; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.asynchttpclient.Request; @@ -73,6 +74,22 @@ public void testBuildControlledClusterFailoverInstance() throws Exception { Assert.assertEquals(request.getHeaders().get(keyB), valueB); } + @Test + public void testInitializeCanOnlyBeCalledOnce() throws Exception { + String defaultServiceUrl = "pulsar://localhost:6650"; + String urlProvider = "http://localhost:8080/test"; + + ServiceUrlProvider provider = ControlledClusterFailover.builder() + .defaultServiceUrl(defaultServiceUrl) + .urlProvider(urlProvider) + .build(); + + try (PulsarClient client = PulsarClient.builder().serviceUrlProvider(provider).build()) { + Throwable error = Assert.expectThrows(IllegalStateException.class, () -> provider.initialize(client)); + Assert.assertEquals(error.getMessage(), "ServiceUrlProvider has already been initialized"); + } + } + @Test public void testControlledClusterFailoverSwitch() throws Exception { String defaultServiceUrl = "pulsar+ssl://localhost:6651"; From 55fd83432bd9735af9c862b625adc471db9ea554 Mon Sep 17 00:00:00 2001 From: void-ptr974 Date: Thu, 4 Jun 2026 02:16:26 +0800 Subject: [PATCH 033/213] [improve][offload] Coalesce automatic offload triggers to reduce retry loops and ledger scans (#25793) (cherry picked from commit 7ecedb8265b1d802ef1a571d7780f4a73141251b) --- .../AutomaticOffloadTriggerController.java | 86 ++++++++++++ .../impl/ManagedLedgerFactoryImpl.java | 4 +- .../mledger/impl/ManagedLedgerImpl.java | 90 +++++++++++-- ...AutomaticOffloadTriggerControllerTest.java | 85 ++++++++++++ .../mledger/impl/OffloadPrefixTest.java | 124 ++++++++++++++++++ 5 files changed, 375 insertions(+), 14 deletions(-) create mode 100644 managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AutomaticOffloadTriggerController.java create mode 100644 managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/AutomaticOffloadTriggerControllerTest.java diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AutomaticOffloadTriggerController.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AutomaticOffloadTriggerController.java new file mode 100644 index 0000000000000..35f5a26706371 --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AutomaticOffloadTriggerController.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.impl; + +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Coalesces repeated automatic offload triggers into at most one active run and one follow-up run. + */ +final class AutomaticOffloadTriggerController { + private static final int IDLE = 0; + private static final int RUNNING = 1; + private static final int RUNNING_WITH_PENDING_TRIGGER = 2; + + private final AtomicInteger state = new AtomicInteger(IDLE); + + /** + * Records an automatic offload trigger. + * + * @return true when the caller must start a new automatic offload run + */ + boolean requestRun() { + while (true) { + int current = state.get(); + switch (current) { + case IDLE: + if (state.compareAndSet(IDLE, RUNNING)) { + return true; + } + break; + case RUNNING: + if (state.compareAndSet(RUNNING, RUNNING_WITH_PENDING_TRIGGER)) { + return false; + } + break; + case RUNNING_WITH_PENDING_TRIGGER: + return false; + default: + throw new IllegalStateException("Unknown automatic offload trigger state: " + current); + } + } + } + + /** + * Records completion of the current automatic offload run. + * + * @return true when the caller must immediately start one coalesced follow-up run + */ + boolean completeRun() { + while (true) { + int current = state.get(); + switch (current) { + case IDLE: + return false; + case RUNNING: + if (state.compareAndSet(RUNNING, IDLE)) { + return false; + } + break; + case RUNNING_WITH_PENDING_TRIGGER: + if (state.compareAndSet(RUNNING_WITH_PENDING_TRIGGER, RUNNING)) { + return true; + } + break; + default: + throw new IllegalStateException("Unknown automatic offload trigger state: " + current); + } + } + } +} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java index bcca3fb9a5286..d45692df78fb6 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java @@ -19,7 +19,7 @@ package org.apache.bookkeeper.mledger.impl; import static org.apache.bookkeeper.mledger.ManagedLedgerException.getManagedLedgerException; -import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.NULL_OFFLOAD_PROMISE; +import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.AUTOMATIC_OFFLOAD_TRIGGER; import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.google.common.base.Predicates; import com.google.common.collect.BoundType; @@ -430,7 +430,7 @@ public void initializeComplete() { future.complete(newledger); // May need to trigger offloading if (config.isTriggerOffloadOnTopicLoad()) { - newledger.maybeOffloadInBackground(NULL_OFFLOAD_PROMISE); + newledger.maybeOffloadInBackground(AUTOMATIC_OFFLOAD_TRIGGER); } }); } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 7c86f4396e39f..31f8aafc18302 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -228,8 +228,20 @@ public class ManagedLedgerImpl implements ManagedLedger, CreateCallback { protected final CallbackMutex trimmerMutex = new CallbackMutex(); protected final CallbackMutex offloadMutex = new CallbackMutex(); - public static final CompletableFuture NULL_OFFLOAD_PROMISE = CompletableFuture + private final AutomaticOffloadTriggerController automaticOffloadTriggerController = + new AutomaticOffloadTriggerController(); + // Identity sentinel for automatic offload requests. The completed Position value is not used. + public static final CompletableFuture AUTOMATIC_OFFLOAD_TRIGGER = CompletableFuture .completedFuture(PositionFactory.LATEST); + + private enum OffloadRequestSource { + AUTOMATIC, + EXPLICIT + } + + private record OffloadThresholds(long thresholdInBytes, long thresholdInSeconds) { + } + @VisibleForTesting @Getter protected volatile LedgerHandle currentLedger; @@ -1955,7 +1967,7 @@ synchronized void ledgerClosed(final LedgerHandle lh, Long lastAddConfirmed, Led trimConsumedLedgersInBackground(); - maybeOffloadInBackground(NULL_OFFLOAD_PROMISE); + maybeOffloadInBackground(AUTOMATIC_OFFLOAD_TRIGGER); createLedgerAfterClosed(ledgerRollReason); } @@ -2846,22 +2858,73 @@ private void scheduleDeferredTrimming(boolean isTruncate, CompletableFuture p } public void maybeOffloadInBackground(CompletableFuture promise) { - if (getOffloadPoliciesIfAppendable().isEmpty()) { + if (promise == AUTOMATIC_OFFLOAD_TRIGGER) { + if (automaticOffloadTriggerController.requestRun()) { + startAutomaticOffload(); + } + return; + } + + maybeOffloadInBackground(promise, OffloadRequestSource.EXPLICIT); + } + + private void startAutomaticOffload() { + CompletableFuture automaticOffloadCompletion = new CompletableFuture<>(); + automaticOffloadCompletion.whenComplete((res, ex) -> finishAutomaticOffload(ex)); + try { + maybeOffloadInBackground(automaticOffloadCompletion, OffloadRequestSource.AUTOMATIC); + } catch (RuntimeException e) { + automaticOffloadCompletion.completeExceptionally(e); + } + } + + private void maybeOffloadInBackground(CompletableFuture promise, OffloadRequestSource source) { + Optional offloadThresholds = getOffloadThresholds(); + if (offloadThresholds.isEmpty()) { + if (source == OffloadRequestSource.AUTOMATIC) { + promise.complete(PositionFactory.LATEST); + } return; } - final OffloadPolicies policies = config.getLedgerOffloader().getOffloadPolicies(); + OffloadThresholds thresholds = offloadThresholds.get(); + try { + executor.execute(() -> maybeOffload(thresholds.thresholdInBytes(), thresholds.thresholdInSeconds(), + promise, source)); + } catch (RuntimeException e) { + promise.completeExceptionally(e); + } + } + + private Optional getOffloadThresholds() { + Optional optionalOffloadPolicies = getOffloadPoliciesIfAppendable(); + if (optionalOffloadPolicies.isEmpty()) { + return Optional.empty(); + } + + final OffloadPolicies policies = optionalOffloadPolicies.get(); final long offloadThresholdInBytes = Optional.ofNullable(policies.getManagedLedgerOffloadThresholdInBytes()).orElse(-1L); final long offloadThresholdInSeconds = Optional.ofNullable(policies.getManagedLedgerOffloadThresholdInSeconds()).orElse(-1L); if (offloadThresholdInBytes >= 0 || offloadThresholdInSeconds >= 0) { - executor.execute(() -> maybeOffload(offloadThresholdInBytes, offloadThresholdInSeconds, promise)); + return Optional.of(new OffloadThresholds(offloadThresholdInBytes, offloadThresholdInSeconds)); + } + + return Optional.empty(); + } + + private void finishAutomaticOffload(Throwable exception) { + if (exception != null && log.isDebugEnabled()) { + log.debug("Failed to automatically offload ledgers", exception); + } + if (automaticOffloadTriggerController.completeRun()) { + startAutomaticOffload(); } } private void maybeOffload(long offloadThresholdInBytes, long offloadThresholdInSeconds, - CompletableFuture finalPromise) { + CompletableFuture finalPromise, OffloadRequestSource source) { if (getOffloadPoliciesIfAppendable().isEmpty()) { String msg = String.format("[%s] Nothing to offload due to offloader or offloadPolicies is NULL", name); finalPromise.completeExceptionally(new IllegalArgumentException(msg)); @@ -2876,8 +2939,12 @@ private void maybeOffload(long offloadThresholdInBytes, long offloadThresholdInS } if (!offloadMutex.tryLock()) { - scheduledExecutor.schedule(() -> maybeOffloadInBackground(finalPromise), - 100, TimeUnit.MILLISECONDS); + try { + scheduledExecutor.schedule(() -> maybeOffloadInBackground(finalPromise, source), + 100, TimeUnit.MILLISECONDS); + } catch (RuntimeException e) { + finalPromise.completeExceptionally(e); + } return; } @@ -2967,12 +3034,11 @@ void internalTrimConsumedLedgers(CompletableFuture promise) { private Optional getOffloadPoliciesIfAppendable() { LedgerOffloader ledgerOffloader = config.getLedgerOffloader(); - if (ledgerOffloader == null - || !ledgerOffloader.isAppendable() - || ledgerOffloader.getOffloadPolicies() == null) { + if (ledgerOffloader == null || !ledgerOffloader.isAppendable()) { return Optional.empty(); } - return Optional.ofNullable(ledgerOffloader.getOffloadPolicies()); + OffloadPolicies offloadPolicies = ledgerOffloader.getOffloadPolicies(); + return Optional.ofNullable(offloadPolicies); } @VisibleForTesting diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/AutomaticOffloadTriggerControllerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/AutomaticOffloadTriggerControllerTest.java new file mode 100644 index 0000000000000..9aefc7e3e1e1f --- /dev/null +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/AutomaticOffloadTriggerControllerTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.impl; + +import static org.assertj.core.api.Assertions.assertThat; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import org.testng.annotations.Test; + +public class AutomaticOffloadTriggerControllerTest { + + @Test + public void triggersCoalesceWhileRunIsActive() { + AutomaticOffloadTriggerController controller = new AutomaticOffloadTriggerController(); + + assertThat(controller.requestRun()).isTrue(); + assertThat(controller.requestRun()).isFalse(); + assertThat(controller.requestRun()).isFalse(); + } + + @Test + public void pendingTriggerSchedulesOneFollowUpRun() { + AutomaticOffloadTriggerController controller = new AutomaticOffloadTriggerController(); + + assertThat(controller.requestRun()).isTrue(); + assertThat(controller.requestRun()).isFalse(); + + assertThat(controller.completeRun()).isTrue(); + assertThat(controller.completeRun()).isFalse(); + assertThat(controller.requestRun()).isTrue(); + } + + @Test(timeOut = 30000) + public void concurrentTriggerAndCompletionAlwaysReserveOneFollowUpRun() throws Exception { + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + for (int i = 0; i < 1000; i++) { + AutomaticOffloadTriggerController controller = new AutomaticOffloadTriggerController(); + assertThat(controller.requestRun()).isTrue(); + + // Completion and a new trigger can race; exactly one side must reserve the follow-up run. + CyclicBarrier barrier = new CyclicBarrier(3); + Future completeResult = executor.submit(() -> { + barrier.await(5, TimeUnit.SECONDS); + return controller.completeRun(); + }); + Future triggerResult = executor.submit(() -> { + barrier.await(5, TimeUnit.SECONDS); + return controller.requestRun(); + }); + + barrier.await(5, TimeUnit.SECONDS); + boolean followUpReservedByComplete = completeResult.get(5, TimeUnit.SECONDS); + boolean followUpReservedByTrigger = triggerResult.get(5, TimeUnit.SECONDS); + + assertThat(followUpReservedByComplete) + .as("iteration %s must reserve exactly one follow-up run", i) + .isNotEqualTo(followUpReservedByTrigger); + assertThat(controller.completeRun()).isFalse(); + } + } finally { + executor.shutdownNow(); + executor.awaitTermination(5, TimeUnit.SECONDS); + } + } +} diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java index 2a50330a0da7c..434980f000762 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java @@ -1184,6 +1184,130 @@ public CompletableFuture offload(ReadHandle ledger, } } + @Test + public void automaticOffloadTriggersAreCoalescedWhileOffloadInProgress() throws Exception { + CompletableFuture slowOffload = new CompletableFuture<>(); + CountDownLatch offloadRunning = new CountDownLatch(1); + AtomicInteger offloadPolicyCalls = new AtomicInteger(); + MockLedgerOffloader offloader = new MockLedgerOffloader() { + @Override + public CompletableFuture offload(ReadHandle ledger, + UUID uuid, + Map extraMetadata) { + offloadRunning.countDown(); + return slowOffload.thenCompose((res) -> super.offload(ledger, uuid, extraMetadata)); + } + + @Override + public OffloadPoliciesImpl getOffloadPolicies() { + offloadPolicyCalls.incrementAndGet(); + return super.getOffloadPolicies(); + } + }; + + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setMaxEntriesPerLedger(10); + config.setRetentionTime(10, TimeUnit.MINUTES); + config.setRetentionSizeInMB(10); + offloader.getOffloadPolicies().setManagedLedgerOffloadThresholdInBytes(0L); + offloader.getOffloadPolicies().setManagedLedgerOffloadThresholdInSeconds(null); + config.setLedgerOffloader(offloader); + + ManagedLedgerImpl ledger = + (ManagedLedgerImpl) factory.open("my_test_ledger" + UUID.randomUUID(), config); + + for (int i = 0; i < 25; i++) { + ledger.addEntry(buildEntry(10, "entry-" + i)); + } + assertTrue(offloadRunning.await(5, TimeUnit.SECONDS)); + + // Repeated automatic triggers should stop at the controller and avoid another policy lookup. + int callsBeforeRepeatedTriggers = offloadPolicyCalls.get(); + for (int i = 0; i < 20; i++) { + ledger.maybeOffloadInBackground(ManagedLedgerImpl.AUTOMATIC_OFFLOAD_TRIGGER); + } + + assertEquals(offloadPolicyCalls.get(), callsBeforeRepeatedTriggers); + + slowOffload.complete(null); + + assertEventuallyTrue(() -> offloader.offloadedLedgers().size() == 2); + List allLedgerIds = ledger.getLedgersInfoAsList().stream().map(LedgerInfo::getLedgerId).toList(); + assertEquals(offloader.offloadedLedgers(), Set.of(allLedgerIds.get(0), allLedgerIds.get(1))); + } + + @Test + public void automaticOffloadRunsAgainForCoalescedTrigger() throws Exception { + CompletableFuture slowOffload = new CompletableFuture<>(); + CountDownLatch offloadRunning = new CountDownLatch(1); + MockLedgerOffloader offloader = new MockLedgerOffloader() { + @Override + public CompletableFuture offload(ReadHandle ledger, + UUID uuid, + Map extraMetadata) { + offloadRunning.countDown(); + return slowOffload.thenCompose((res) -> super.offload(ledger, uuid, extraMetadata)); + } + }; + + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setMaxEntriesPerLedger(10); + config.setRetentionTime(10, TimeUnit.MINUTES); + config.setRetentionSizeInMB(10); + offloader.getOffloadPolicies().setManagedLedgerOffloadThresholdInBytes(0L); + offloader.getOffloadPolicies().setManagedLedgerOffloadThresholdInSeconds(null); + config.setLedgerOffloader(offloader); + + ManagedLedgerImpl ledger = + (ManagedLedgerImpl) factory.open("my_test_ledger" + UUID.randomUUID(), config); + + for (int i = 0; i < 11; i++) { + ledger.addEntry(buildEntry(10, "entry-" + i)); + } + assertTrue(offloadRunning.await(5, TimeUnit.SECONDS)); + + // The next ledger closes after the first automatic scan, so it depends on the coalesced rerun. + for (int i = 11; i < 21; i++) { + ledger.addEntry(buildEntry(10, "entry-" + i)); + } + assertEquals(offloader.offloadedLedgers().size(), 0); + + slowOffload.complete(null); + + assertEventuallyTrue(() -> offloader.offloadedLedgers().size() == 2); + List allLedgerIds = ledger.getLedgersInfoAsList().stream().map(LedgerInfo::getLedgerId).toList(); + assertEquals(offloader.offloadedLedgers(), Set.of(allLedgerIds.get(0), allLedgerIds.get(1))); + } + + @Test + public void automaticOffloadWithoutThresholdDoesNotBlockLaterTriggers() throws Exception { + MockLedgerOffloader offloader = new MockLedgerOffloader(); + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setMaxEntriesPerLedger(10); + config.setRetentionTime(10, TimeUnit.MINUTES); + config.setRetentionSizeInMB(10); + offloader.getOffloadPolicies().setManagedLedgerOffloadThresholdInBytes(-1L); + offloader.getOffloadPolicies().setManagedLedgerOffloadThresholdInSeconds(null); + config.setLedgerOffloader(offloader); + + ManagedLedgerImpl ledger = + (ManagedLedgerImpl) factory.open("my_test_ledger" + UUID.randomUUID(), config); + + for (int i = 0; i < 25; i++) { + ledger.addEntry(buildEntry(10, "entry-" + i)); + } + ledger.maybeOffloadInBackground(ManagedLedgerImpl.AUTOMATIC_OFFLOAD_TRIGGER); + assertEquals(offloader.offloadedLedgers().size(), 0); + + // A disabled automatic trigger must complete internally so a later valid trigger can run. + offloader.getOffloadPolicies().setManagedLedgerOffloadThresholdInBytes(0L); + ledger.maybeOffloadInBackground(ManagedLedgerImpl.AUTOMATIC_OFFLOAD_TRIGGER); + + assertEventuallyTrue(() -> offloader.offloadedLedgers().size() == 2); + List allLedgerIds = ledger.getLedgersInfoAsList().stream().map(LedgerInfo::getLedgerId).toList(); + assertEquals(offloader.offloadedLedgers(), Set.of(allLedgerIds.get(0), allLedgerIds.get(1))); + } + @DataProvider(name = "offloadAsSoonAsClosed") public Object[][] offloadAsSoonAsClosedProvider() { return new Object[][]{ From 5378869a69660a687334f586e62705cca8aba8ed Mon Sep 17 00:00:00 2001 From: Oneby Wang <44369297+oneby-wang@users.noreply.github.com> Date: Thu, 4 Jun 2026 03:50:24 +0800 Subject: [PATCH 034/213] [fix][test] Stabilize testSecondaryIsolationGroupsBookiesNegative() test (#25900) (cherry picked from commit 7ab09413c8bccaa759ce5efcd187cd8e7bd81580) --- .../IsolatedBookieEnsemblePlacementPolicyTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java index fc67395c813c4..d4785d6ca83d5 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java @@ -595,6 +595,12 @@ public void testSecondaryIsolationGroupsBookiesNegative() throws Exception { NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); isolationPolicy.onClusterChanged(writableBookies, readOnlyBookies); + // Wait for the async cache load triggered by initialize() to complete; otherwise the + // first newEnsemble call can race with an empty cachedRackConfiguration and skip isolation. + Awaitility.await().atMost(Duration.ofSeconds(5)).untilAsserted(() -> + assertNotNull(isolationPolicy.getBookieMappingCache() + .getIfCached(BookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH))); + try { isolationPolicy .newEnsemble(3, 3, 2, Collections.emptyMap(), new HashSet<>()).getResult(); From 27544e154743a9c1cde62e0ae88b17c0bf63e2b8 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 4 Jun 2026 09:21:18 +0300 Subject: [PATCH 035/213] [improve][misc] Upgrade vert.x to 4.5.28 (#25924) (cherry picked from commit 1bbe964056669315507b0c12f828ec6d3c671f1e) --- distribution/server/src/assemble/LICENSE.bin.txt | 10 +++++----- pom.xml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index b26c3534ec7a8..d326a94353366 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -508,11 +508,11 @@ The Apache Software License, Version 2.0 * JCTools - Java Concurrency Tools for the JVM - org.jctools-jctools-core-2.1.2.jar * Vertx - - io.vertx-vertx-auth-common-4.5.27.jar - - io.vertx-vertx-bridge-common-4.5.27.jar - - io.vertx-vertx-core-4.5.27.jar - - io.vertx-vertx-web-4.5.27.jar - - io.vertx-vertx-web-common-4.5.27.jar + - io.vertx-vertx-auth-common-4.5.28.jar + - io.vertx-vertx-bridge-common-4.5.28.jar + - io.vertx-vertx-core-4.5.28.jar + - io.vertx-vertx-web-4.5.28.jar + - io.vertx-vertx-web-common-4.5.28.jar * Apache ZooKeeper - org.apache.zookeeper-zookeeper-jute-3.9.5.jar * Snappy Java diff --git a/pom.xml b/pom.xml index 46faf2109a7b0..1c364712d7dff 100644 --- a/pom.xml +++ b/pom.xml @@ -196,7 +196,7 @@ flexible messaging model and an intuitive client API. 2.42 1.10.62 0.16.0 - 4.5.27 + 4.5.28 7.9.2 2.0.17 4.5.0 From 067e4328e28ba93243fa6092df97ea35116cdf9c Mon Sep 17 00:00:00 2001 From: iantowey <44482548+iantowey@users.noreply.github.com> Date: Wed, 3 Jun 2026 21:00:08 +0100 Subject: [PATCH 036/213] [improve][functions] Allow customizing Kubernetes service domain suffix in Function Worker (#25872) Co-authored-by: Ian (cherry picked from commit cc9fddcd02939ba6c735d88a2bfe702a22fb2ff4) --- conf/functions_worker.yml | 3 +++ .../runtime/kubernetes/KubernetesRuntime.java | 12 ++++++---- .../kubernetes/KubernetesRuntimeFactory.java | 4 +++- .../KubernetesRuntimeFactoryConfig.java | 7 +++++- .../kubernetes/KubernetesRuntimeTest.java | 23 +++++++++++++++++++ 5 files changed, 42 insertions(+), 7 deletions(-) diff --git a/conf/functions_worker.yml b/conf/functions_worker.yml index 6f995576ebd64..794cfba1d519b 100644 --- a/conf/functions_worker.yml +++ b/conf/functions_worker.yml @@ -198,6 +198,9 @@ functionRuntimeFactoryConfigs: # # The Kubernetes pod name to run the function instances. It is set to # # `pf----` if this setting is left to be empty # jobName: +# # Optional domain suffix to use when the Function Worker constructs the gRPC address to connect to function instances. +# # If left blank, it defaults to `.svc.cluster.local`. Set this if your Function Worker is outside the cluster and connects via an external Gateway/Ingress. +# kubernetesServiceDomainSuffix: # # the docker image to run function instance. by default it is `apachepulsar/pulsar` # pulsarDockerImageName: # # the docker image to run function instance according to different configurations provided by users. diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java index 7a69b822cbd89..3e460f7b97ace 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java @@ -153,11 +153,13 @@ public class KubernetesRuntime implements Runtime { private final Optional manifestCustomizer; private String functionInstanceClassPath; private String downloadDirectory; + private final String kubernetesServiceDomainSuffix; KubernetesRuntime(AppsV1Api appsClient, CoreV1Api coreClient, String jobNamespace, String jobName, + String kubernetesServiceDomainSuffix, Map customLabels, Boolean installUserCodeDependencies, String pythonDependencyRepository, @@ -196,6 +198,7 @@ public class KubernetesRuntime implements Runtime { this.instanceConfig = instanceConfig; this.jobNamespace = jobNamespace; this.jobName = jobName; + this.kubernetesServiceDomainSuffix = kubernetesServiceDomainSuffix; this.customLabels = customLabels; this.functionDockerImages = functionDockerImages; this.pulsarDockerImageName = pulsarDockerImageName; @@ -320,7 +323,6 @@ private synchronized void setupGrpcChannelIfNeeded() { channel = new ManagedChannel[instanceConfig.getFunctionDetails().getParallelism()]; stub = new InstanceControlGrpc.InstanceControlFutureStub[instanceConfig.getFunctionDetails() .getParallelism()]; - String jobName = createJobName(instanceConfig.getFunctionDetails(), this.jobName); for (int i = 0; i < instanceConfig.getFunctionDetails().getParallelism(); ++i) { String address = getServiceUrl(jobName, jobNamespace, i); @@ -1194,11 +1196,11 @@ private static String createJobName(String tenant, String namespace, String func final String shortHash = DigestUtils.sha1Hex(jobNameBase).toLowerCase().substring(0, 8); return convertedJobName + "-" + shortHash; } - - private static String getServiceUrl(String jobName, String jobNamespace, int instanceId) { - return String.format("%s-%d.%s.%s.svc.cluster.local", jobName, instanceId, jobName, jobNamespace); + @VisibleForTesting + String getServiceUrl(String jobName, String jobNamespace, int instanceId) { + String suffix = isNotBlank(kubernetesServiceDomainSuffix) ? kubernetesServiceDomainSuffix : "svc.cluster.local"; + return String.format("%s-%d.%s.%s.%s", jobName, instanceId, jobName, jobNamespace, suffix); } - public static void doChecks(Function.FunctionDetails functionDetails, String overridenJobName) { final String jobName = createJobName(functionDetails, overridenJobName); if (!jobName.equals(jobName.toLowerCase())) { diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java index bbb6e3992a018..cba13acd5b037 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java @@ -101,6 +101,7 @@ public class KubernetesRuntimeFactory implements RuntimeFactory { private String functionInstanceClassPath; private String downloadDirectory; private int gracePeriodSeconds; + private String kubernetesServiceDomainSuffix; @ToString.Exclude @EqualsAndHashCode.Exclude @@ -178,7 +179,7 @@ public void initialize(WorkerConfig workerConfig, AuthenticationConfig authentic if (!Paths.get(this.downloadDirectory).isAbsolute()) { this.downloadDirectory = this.pulsarRootDir + "/" + this.downloadDirectory; } - + this.kubernetesServiceDomainSuffix = factoryConfig.getKubernetesServiceDomainSuffix(); this.submittingInsidePod = factoryConfig.getSubmittingInsidePod(); this.installUserCodeDependencies = factoryConfig.getInstallUserCodeDependencies(); this.pythonDependencyRepository = factoryConfig.getPythonDependencyRepository(); @@ -318,6 +319,7 @@ public KubernetesRuntime createContainer(InstanceConfig instanceConfig, String c // get the namespace for this function overriddenNamespace, overriddenName, + kubernetesServiceDomainSuffix, customLabels, installUserCodeDependencies, pythonDependencyRepository, diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryConfig.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryConfig.java index 43cdc035076c3..ea19692349794 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryConfig.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryConfig.java @@ -47,7 +47,12 @@ public class KubernetesRuntimeFactoryConfig { doc = "The docker image used to run function instance. By default it is `apachepulsar/pulsar`" ) protected String pulsarDockerImageName; - + @FieldContext( + doc = "Optional domain suffix to use when the Function Worker constructs the gRPC address " + + "to connect to function instances. If left blank, it defaults to `.svc.cluster.local`. " + + "Set this if your Function Worker is outside the cluster and connects via an external Gateway/Ingress." + ) + protected String kubernetesServiceDomainSuffix; @FieldContext( doc = "The function docker images used to run function instance according to different " + "configurations provided by users. By default it is `apachepulsar/pulsar`" diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java index f8069efe299cb..585c7e920095b 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java @@ -341,6 +341,29 @@ InstanceConfig createJavaInstanceConfig(FunctionDetails.Runtime runtime, boolean return config; } + @Test + public void testGetServiceUrl() throws Exception { + factory = createKubernetesRuntimeFactory(null, 10, 1.0, 1.0); + InstanceConfig config = createJavaInstanceConfig(FunctionDetails.Runtime.JAVA, true); + + KubernetesRuntime container1 = factory.createContainer( + config, userJarFile, userJarFile, null, null, 30L); + assertEquals(container1.getServiceUrl("my-job", "my-namespace", 0), + "my-job-0.my-job.my-namespace.svc.cluster.local"); + + KubernetesRuntimeFactory factory2 = createKubernetesRuntimeFactory(null, 10, 1.0, 1.0); + java.lang.reflect.Field field = KubernetesRuntimeFactory.class.getDeclaredField( + "kubernetesServiceDomainSuffix"); + field.setAccessible(true); + field.set(factory2, "custom.gateway.internal"); + + KubernetesRuntime container2 = factory2.createContainer( + config, userJarFile, userJarFile, null, null, 30L); + assertEquals(container2.getServiceUrl("my-job", "my-namespace", 0), + "my-job-0.my-job.my-namespace.custom.gateway.internal"); + } + + @Test public void testRamPadding() throws Exception { verifyRamPadding(0, 1000, 1000); From 77790df2b627bac520e5a2aee93d3ba0902a9862 Mon Sep 17 00:00:00 2001 From: Dream95 <864197662@qq.com> Date: Fri, 5 Jun 2026 18:33:10 +0800 Subject: [PATCH 037/213] [fix][client] Match logical topic when removing unacked messages (#25921) Signed-off-by: Dream95 (cherry picked from commit 4e509574e5c50c3edc872dd5a2c65c1b0c35909b) --- .../pulsar/client/api/TopicMessageId.java | 13 ++++++ .../client/impl/TopicMessageIdImpl.java | 7 ++++ .../UnAckedTopicMessageRedeliveryTracker.java | 4 +- .../impl/UnAckedTopicMessageTracker.java | 2 +- .../client/impl/TopicMessageIdImplTest.java | 17 ++++++++ ...ckedTopicMessageRedeliveryTrackerTest.java | 40 ++++++++++++++++++- 6 files changed, 79 insertions(+), 4 deletions(-) diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java index 4d02a7f4096d6..e2791ebf26a9f 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java @@ -41,6 +41,19 @@ public interface TopicMessageId extends MessageId { */ String getOwnerTopic(); + /** + * Checks if this message's owner topic and the given topic refer to the same base + * partitioned topic by comparing their base partitioned topic names. + * + *

For example, {@code persistent://public/default/my-topic-partition-0} matches + * {@code persistent://public/default/my-topic} or any other partition of that topic. + * Topics sharing only a name prefix (e.g., {@code my-topic} vs {@code my-topic-v2}) do not match. + * + * @param topicName a full topic name (non-partitioned, partitioned, or specific partition) + * @return {@code true} if both topics resolve to the same base partitioned topic name + */ + boolean hasSameBasePartitionedTopic(String topicName); + static TopicMessageId create(String topic, MessageId messageId) { if (messageId instanceof TopicMessageId) { return (TopicMessageId) messageId; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java index 3dc9b23e93e86..a75b7278666ae 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java @@ -22,6 +22,7 @@ import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.TopicMessageId; +import org.apache.pulsar.common.naming.TopicName; public class TopicMessageIdImpl implements MessageIdAdv, TopicMessageId { @@ -90,6 +91,12 @@ public String getOwnerTopic() { return ownerTopic; } + @Override + public boolean hasSameBasePartitionedTopic(String topicName) { + return TopicName.get(getOwnerTopic()).getPartitionedTopicName() + .equals(TopicName.get(topicName).getPartitionedTopicName()); + } + @Override public long getLedgerId() { return msgId.getLedgerId(); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTracker.java index 393557cd89adb..a13b636e3ec78 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTracker.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTracker.java @@ -43,7 +43,7 @@ public int removeTopicMessages(String topicName) { UnackMessageIdWrapper messageIdWrapper = entry.getKey(); MessageId messageId = messageIdWrapper.getMessageId(); if (messageId instanceof TopicMessageId - && ((TopicMessageId) messageId).getOwnerTopic().contains(topicName)) { + && ((TopicMessageId) messageId).hasSameBasePartitionedTopic(topicName)) { entry.getValue().remove(messageIdWrapper); iterator.remove(); messageIdWrapper.recycle(); @@ -55,7 +55,7 @@ public int removeTopicMessages(String topicName) { while (iteratorAckTimeOut.hasNext()) { MessageId messageId = iteratorAckTimeOut.next(); if (messageId instanceof TopicMessageId - && ((TopicMessageId) messageId).getOwnerTopic().contains(topicName)) { + && ((TopicMessageId) messageId).hasSameBasePartitionedTopic(topicName)) { iteratorAckTimeOut.remove(); removed++; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedTopicMessageTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedTopicMessageTracker.java index 1cbab5844046f..669a2ed961ba7 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedTopicMessageTracker.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedTopicMessageTracker.java @@ -41,7 +41,7 @@ public int removeTopicMessages(String topicName) { Entry> entry = iterator.next(); MessageId messageId = entry.getKey(); if (messageId instanceof TopicMessageId - && ((TopicMessageId) messageId).getOwnerTopic().contains(topicName)) { + && ((TopicMessageId) messageId).hasSameBasePartitionedTopic(topicName)) { entry.getValue().remove(messageId); iterator.remove(); removed++; diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java index 59f15266c1091..26f4335b75a20 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java @@ -19,8 +19,10 @@ package org.apache.pulsar.client.impl; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertSame; +import static org.testng.Assert.assertTrue; import org.testng.annotations.Test; public class TopicMessageIdImplTest { @@ -54,6 +56,21 @@ public void equalsTest() { assertNotEquals(topicMsgId1, topicMsgId2); } + @Test + public void testHasSameBasePartitionedTopic() { + MessageIdImpl msgId = new MessageIdImpl(0, 0, 0); + TopicMessageIdImpl partitionMsgId = new TopicMessageIdImpl( + "persistent://public/default/my-topic-partition-0", msgId); + assertTrue(partitionMsgId.hasSameBasePartitionedTopic( + "persistent://public/default/my-topic-partition-1")); + assertTrue(partitionMsgId.hasSameBasePartitionedTopic( + "persistent://public/default/my-topic")); + assertFalse(partitionMsgId.hasSameBasePartitionedTopic( + "persistent://public/default/my-topic-v2")); + assertFalse(partitionMsgId.hasSameBasePartitionedTopic( + "persistent://public/default/my-topic-v2-partition-0")); + } + @Test public void testDeprecatedMethods() { BatchMessageIdImpl msgId = new BatchMessageIdImpl(1, 2, 3, 4); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTrackerTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTrackerTest.java index 9fdab39145bd8..e0ad1f9111b7e 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTrackerTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedTopicMessageRedeliveryTrackerTest.java @@ -70,10 +70,48 @@ public void testRemoveTopicMessages() { tracker.ackTimeoutMessages.put(msgInAckTimeout, System.currentTimeMillis() + 1_000_000L); assertEquals(tracker.size(), 2); - assertEquals(tracker.removeTopicMessages("my-topic"), 2); + assertEquals(tracker.removeTopicMessages("persistent://public/default/my-topic-partition-0"), 2); assertTrue(tracker.isEmpty()); tracker.close(); } + @Test + @SuppressWarnings("unchecked") + public void testRemoveTopicMessagesDoesNotMatchPrefixTopic() { + PulsarClientImpl client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.instrumentProvider()).thenReturn(InstrumentProvider.NOOP); + when(client.getCnxPool()).thenReturn(connectionPool); + @Cleanup("stop") + Timer timer = new HashedWheelTimer( + new DefaultThreadFactory("pulsar-timer", Thread.currentThread().isDaemon()), + 1, TimeUnit.MILLISECONDS); + when(client.timer()).thenReturn(timer); + + ConsumerBase consumer = mock(ConsumerBase.class); + doNothing().when(consumer).onAckTimeoutSend(any()); + doNothing().when(consumer).redeliverUnacknowledgedMessages(any()); + + ConsumerConfigurationData conf = new ConsumerConfigurationData<>(); + conf.setAckTimeoutMillis(1_000_000); + conf.setTickDurationMillis(100_000); + conf.setAckTimeoutRedeliveryBackoff(MultiplierRedeliveryBackoff.builder().build()); + + UnAckedTopicMessageRedeliveryTracker tracker = + new UnAckedTopicMessageRedeliveryTracker(client, consumer, conf); + + String ownerTopic = "persistent://public/default/my-topic-v2-partition-0"; + TopicMessageIdImpl msgInPartition = + new TopicMessageIdImpl(ownerTopic, new MessageIdImpl(1L, 0L, -1)); + + assertTrue(tracker.add(msgInPartition)); + assertEquals(tracker.size(), 1); + + assertEquals(tracker.removeTopicMessages("persistent://public/default/my-topic"), 0); + assertEquals(tracker.size(), 1); + + tracker.close(); + } + } From eb3bf782fa419fe09482723e98378878f533dce9 Mon Sep 17 00:00:00 2001 From: Dream95 <864197662@qq.com> Date: Fri, 5 Jun 2026 18:33:24 +0800 Subject: [PATCH 038/213] [improve][client] Clean up unacked message tracker when topics are removed in multi-topic consumers (#25923) Signed-off-by: Dream95 (cherry picked from commit 26cf550bb9635a8107e5628800b465778c847a91) --- .../client/impl/MultiTopicsConsumerImpl.java | 17 ++++--- .../impl/PatternMultiTopicsConsumerImpl.java | 1 + .../impl/MultiTopicsConsumerImplTest.java | 51 +++++++++++++++++++ .../PatternMultiTopicsConsumerImplTest.java | 41 +++++++++++++++ 4 files changed, 104 insertions(+), 6 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index 560d62851aa89..b3bc3f6a46f10 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -968,6 +968,14 @@ private void removeTopic(String topic) { } } + protected void removeTopicMessagesFromUnackedTracker(String topicName) { + if (unAckedMessageTracker instanceof UnAckedTopicMessageTracker) { + ((UnAckedTopicMessageTracker) unAckedMessageTracker).removeTopicMessages(topicName); + } else if (unAckedMessageTracker instanceof UnAckedTopicMessageRedeliveryTracker) { + ((UnAckedTopicMessageRedeliveryTracker) unAckedMessageTracker).removeTopicMessages(topicName); + } + } + /*** * Subscribe one more given topic. * @param topicName topic name without the partition suffix. @@ -1302,11 +1310,7 @@ public CompletableFuture unsubscribeAsync(String topicName) { }); removeTopic(topicName); - if (unAckedMessageTracker instanceof UnAckedTopicMessageTracker) { - ((UnAckedTopicMessageTracker) unAckedMessageTracker).removeTopicMessages(topicName); - } else if (unAckedMessageTracker instanceof UnAckedTopicMessageRedeliveryTracker) { - ((UnAckedTopicMessageRedeliveryTracker) unAckedMessageTracker).removeTopicMessages(topicName); - } + removeTopicMessagesFromUnackedTracker(topicName); unsubscribeFuture.complete(null); log.info("[{}] [{}] [{}] Unsubscribed Topics Consumer, allTopicPartitionsNumber: {}", @@ -1433,7 +1437,8 @@ private CompletableFuture subscribeIncreasedTopicPartitions(String topicNa } } - return FutureUtil.waitForAll(futures); + return FutureUtil.waitForAll(futures) + .thenRun(() -> removeTopicMessagesFromUnackedTracker(topicName)); } else if (oldPartitionNumber < currentPartitionNumber) { allTopicPartitionsNumber.addAndGet(currentPartitionNumber - oldPartitionNumber); partitionedTopics.put(topicName, currentPartitionNumber); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImpl.java index 3da14637a1a6f..c0da46c13c77a 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImpl.java @@ -366,6 +366,7 @@ public CompletableFuture onTopicsRemoved(Collection removedTopics) removedPartitionedTopicsForLog.add(String.format("%s with %s partitions", groupedTopicRemoved, partitions)); partitionedTopics.remove(groupedTopicRemoved, partitions); + removeTopicMessagesFromUnackedTracker(groupedTopicRemoved); } } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java index 54175613e3bb8..b4d007d855f88 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java @@ -37,6 +37,7 @@ import io.netty.util.Timer; import io.netty.util.concurrent.DefaultThreadFactory; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; @@ -266,4 +267,54 @@ public void testDontCheckForPartitionsUpdatesOnNonPartitionedTopics() throws Exc verify(clientMock, times(3)).getPartitionedTopicMetadata(any(), anyBoolean(), anyBoolean()); } + @Test + @SuppressWarnings("unchecked") + public void testOnTopicsExtendedRemovedTopicCleansUnackedMessages() { + String topicName = "persistent://public/default/deleted-topic"; + String topicPartition0 = topicName + "-partition-0"; + String topicPartition1 = topicName + "-partition-1"; + String otherTopicPartition = "persistent://public/default/other-topic-partition-0"; + + ConsumerConfigurationData consumerConfData = new ConsumerConfigurationData<>(); + consumerConfData.setSubscriptionName("subscriptionName"); + consumerConfData.setAutoUpdatePartitions(true); + consumerConfData.setAutoUpdatePartitionsIntervalSeconds(60); + consumerConfData.setAckTimeoutMillis(1000); + + MultiTopicsConsumerImpl impl = createMultiTopicsConsumer(consumerConfData); + + impl.partitionedTopics.put(topicName, 2); + impl.allTopicPartitionsNumber.set(2); + + ConsumerImpl partitionConsumer0 = (ConsumerImpl) mock(ConsumerImpl.class); + ConsumerImpl partitionConsumer1 = (ConsumerImpl) mock(ConsumerImpl.class); + when(partitionConsumer0.getTopic()).thenReturn(topicPartition0); + when(partitionConsumer1.getTopic()).thenReturn(topicPartition1); + when(partitionConsumer0.closeAsync()).thenReturn(CompletableFuture.completedFuture(null)); + when(partitionConsumer1.closeAsync()).thenReturn(CompletableFuture.completedFuture(null)); + impl.consumers.put(topicPartition0, partitionConsumer0); + impl.consumers.put(topicPartition1, partitionConsumer1); + + TopicMessageIdImpl removedTopicMessageId = new TopicMessageIdImpl(topicPartition0, new MessageIdImpl(1, 1, 0)); + TopicMessageIdImpl otherTopicMessageId = + new TopicMessageIdImpl(otherTopicPartition, new MessageIdImpl(2, 2, 0)); + impl.getUnAckedMessageTracker().add(removedTopicMessageId); + impl.getUnAckedMessageTracker().add(otherTopicMessageId); + assertEquals(impl.getUnAckedMessageTracker().size(), 2); + + when(impl.client.getPartitionsForTopic(topicName, false)).thenReturn(CompletableFuture.completedFuture( + Collections.emptyList())); + + PartitionsChangedListener listener = impl.topicsPartitionChangedListener; + listener.onTopicsExtended(Collections.singleton(topicName)).join(); + + assertTrue(impl.getConsumers().isEmpty()); + assertEquals(impl.partitionedTopics.get(topicName), Integer.valueOf(0)); + assertEquals(impl.allTopicPartitionsNumber.get(), 0); + assertEquals(impl.getUnAckedMessageTracker().size(), 1); + assertTrue(impl.getUnAckedMessageTracker().remove(otherTopicMessageId)); + verify(partitionConsumer0).closeAsync(); + verify(partitionConsumer1).closeAsync(); + } + } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImplTest.java index dd4bbff757acd..fe3338afa7e1e 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImplTest.java @@ -310,6 +310,47 @@ public void testPatternSubscribeAndHashHandlingWithChanges() throws Exception { assertThat(invocationCount.get()).isEqualTo(5); } + @Test + @SuppressWarnings("unchecked") + public void testOnTopicsRemovedCleansUnackedMessagesForRemovedPartitionedTopic() { + String partitionedTopic = "persistent://tenant/namespace/deleted-topic"; + String partition0 = partitionedTopic + "-partition-0"; + String partition1 = partitionedTopic + "-partition-1"; + String otherTopicPartition = "persistent://tenant/namespace/other-topic-partition-0"; + TopicsPattern topicsPattern = + TopicsPatternFactory.create("persistent://tenant/namespace/.*", TopicsPattern.RegexImplementation.JDK); + ConsumerConfigurationData consumerConfData = createConsumerConfigurationData(); + consumerConfData.setAckTimeoutMillis(1000); + + PatternMultiTopicsConsumerImpl consumer = + createPatternMultiTopicsConsumer(consumerConfData, topicsPattern); + + consumer.partitionedTopics.put(partitionedTopic, 2); + + ConsumerImpl partitionConsumer0 = (ConsumerImpl) mock(ConsumerImpl.class); + ConsumerImpl partitionConsumer1 = (ConsumerImpl) mock(ConsumerImpl.class); + when(partitionConsumer0.closeAsync()).thenReturn(CompletableFuture.completedFuture(null)); + when(partitionConsumer1.closeAsync()).thenReturn(CompletableFuture.completedFuture(null)); + consumer.consumers.put(partition0, partitionConsumer0); + consumer.consumers.put(partition1, partitionConsumer1); + + TopicMessageIdImpl removedTopicMessageId = new TopicMessageIdImpl(partition0, new MessageIdImpl(1, 1, 0)); + TopicMessageIdImpl otherTopicMessageId = + new TopicMessageIdImpl(otherTopicPartition, new MessageIdImpl(2, 2, 0)); + consumer.getUnAckedMessageTracker().add(removedTopicMessageId); + consumer.getUnAckedMessageTracker().add(otherTopicMessageId); + assertThat(consumer.getUnAckedMessageTracker().size()).isEqualTo(2); + + consumer.topicsChangeListener.onTopicsRemoved(Arrays.asList(partition0, partition1)).join(); + + assertThat(consumer.partitionedTopics.containsKey(partitionedTopic)).isFalse(); + assertThat(consumer.consumers).doesNotContainKeys(partition0, partition1); + assertThat(consumer.getUnAckedMessageTracker().size()).isEqualTo(1); + assertThat(consumer.getUnAckedMessageTracker().remove(otherTopicMessageId)).isTrue(); + verify(partitionConsumer0).closeAsync(); + verify(partitionConsumer1).closeAsync(); + } + private static void runTimerTasks(Deque tasks) throws Exception { // first drain the queue to a list to avoid an infinite loop List taskList = new ArrayList<>(); From 2facacd71760db9c94231cd86fb3794c2d30a2ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AE=B6=E5=90=8D?= Date: Fri, 5 Jun 2026 18:39:05 +0800 Subject: [PATCH 039/213] [fix][client] Preserve equals in FieldParser map values (#25907) (cherry picked from commit f0a3149abedaf0bf0497a9e15134444569de2b16) --- .../org/apache/pulsar/common/util/FieldParser.java | 2 +- .../apache/pulsar/common/util/FieldParserTest.java | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java index 10c1951ab208b..d6aa2c876b1a7 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/FieldParser.java @@ -346,7 +346,7 @@ private static Map stringToMap(String strValue, Class keyType, C String[] tokens = trim(strValue).split(","); Map map = new HashMap<>(); for (String token : tokens) { - String[] keyValue = trim(token).split("="); + String[] keyValue = trim(token).split("=", 2); checkArgument(keyValue.length == 2, strValue + " map-value is not in correct format key1=value,key2=value2"); map.put(convert(trim(keyValue[0]), keyType), convert(trim(keyValue[1]), valueType)); diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/FieldParserTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/FieldParserTest.java index b22170fa46505..1f9a1e2688267 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/FieldParserTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/FieldParserTest.java @@ -95,6 +95,18 @@ public static class MyConfig { public Set stringSet; } + @Test + public void testMapWithEqualsSignAndEmptyValue() { + Map properties = new HashMap<>(); + properties.put("stringStringMap", "key1=value=1,key2="); + + MyConfig config = new MyConfig(); + FieldParser.update(properties, config); + + assertEquals(config.stringStringMap.get("key1"), "value=1"); + assertEquals(config.stringStringMap.get("key2"), ""); + } + @Test public void testNullStrValue() throws Exception { class TestMap { From 5ad246cf6b37ef4b9262e09fc4c04fe80444eb44 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 5 Jun 2026 16:35:22 +0300 Subject: [PATCH 040/213] [fix][fn] Fix orphan exclusive producer on creation timeout in WorkerUtils.createExclusiveProducerWithRetry (#25942) (cherry picked from commit 2177b0e44d96e87a698c40d47ebb763a9523f2ee) --- .../pulsar/functions/worker/WorkerUtils.java | 11 ++- .../functions/worker/WorkerUtilsTest.java | 69 +++++++++++++++++++ 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerUtils.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerUtils.java index af1edf5c8e80d..c86c6e9a193cc 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerUtils.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerUtils.java @@ -30,8 +30,8 @@ import java.net.URI; import java.nio.file.Files; import java.util.Map; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @@ -60,6 +60,7 @@ import org.apache.pulsar.common.functions.WorkerInfo; import org.apache.pulsar.common.policies.data.FunctionInstanceStatsDataImpl; import org.apache.pulsar.common.policies.data.FunctionInstanceStatsImpl; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.functions.proto.InstanceCommunication; import org.apache.pulsar.functions.runtime.Runtime; import org.apache.pulsar.functions.runtime.RuntimeSpawner; @@ -400,13 +401,17 @@ public static Producer createExclusiveProducerWithRetry(PulsarClient cli int tries = 0; do { try { - return client.newProducer().topic(topic) + CompletableFuture> producerFuture = client.newProducer().topic(topic) .accessMode(ProducerAccessMode.Exclusive) .enableBatching(false) .blockIfQueueFull(true) .compressionType(CompressionType.LZ4) .producerName(producerName) - .createAsync().get(10, TimeUnit.SECONDS); + .createAsync(); + return FutureUtil.getAndCleanupOnInterrupt(producerFuture, Producer::closeAsync); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw e; } catch (Exception e) { log.info("Encountered exception while at creating exclusive producer to topic {}", topic, e); } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/WorkerUtilsTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/WorkerUtilsTest.java index 0f5fca4a8a5a3..823b645c672a3 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/WorkerUtilsTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/WorkerUtilsTest.java @@ -18,26 +18,33 @@ */ package org.apache.pulsar.functions.worker; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import java.io.IOException; import java.net.URISyntaxException; import java.net.URL; import java.util.HashSet; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; +import lombok.Cleanup; import org.apache.distributedlog.DistributedLogConfiguration; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.client.api.CompressionType; @@ -108,6 +115,68 @@ public Boolean get() { } } + @Test + @SuppressWarnings("unchecked") + public void testCreateExclusiveProducerWithRetryClosesProducerOnInterrupt() throws Exception { + Producer producer = mock(Producer.class); + when(producer.closeAsync()).thenReturn(CompletableFuture.completedFuture(null)); + + // producer creation stays pending until the test completes it explicitly + CompletableFuture> producerFuture = new CompletableFuture<>(); + CountDownLatch createAsyncCalled = new CountDownLatch(1); + + ProducerBuilder builder = mock(ProducerBuilder.class); + when(builder.topic(anyString())).thenReturn(builder); + when(builder.producerName(anyString())).thenReturn(builder); + when(builder.enableBatching(anyBoolean())).thenReturn(builder); + when(builder.blockIfQueueFull(anyBoolean())).thenReturn(builder); + when(builder.compressionType(any(CompressionType.class))).thenReturn(builder); + when(builder.accessMode(any())).thenReturn(builder); + when(builder.createAsync()).thenAnswer(invocation -> { + createAsyncCalled.countDown(); + return producerFuture; + }); + + PulsarClient pulsarClient = mock(PulsarClient.class); + when(pulsarClient.newProducer()).thenReturn(builder); + + AtomicReference thrown = new AtomicReference<>(); + AtomicBoolean interruptStatusPreserved = new AtomicBoolean(); + @Cleanup("interrupt") + Thread caller = new Thread(() -> { + try { + WorkerUtils.createExclusiveProducerWithRetry(pulsarClient, "test-topic", "test-producer", + () -> true, 0); + } catch (Throwable t) { + thrown.set(t); + interruptStatusPreserved.set(Thread.currentThread().isInterrupted()); + } + }); + caller.setDaemon(true); + caller.start(); + assertTrue(createAsyncCalled.await(10, TimeUnit.SECONDS)); + + // interrupt the caller while it is waiting for the producer to be created + caller.interrupt(); + caller.join(TimeUnit.SECONDS.toMillis(10)); + assertThat(caller.isAlive()) + .as("Interrupt should abort the retry loop instead of retrying") + .isFalse(); + + assertThat(thrown.get()) + .isInstanceOf(RuntimeException.class) + .hasCauseInstanceOf(InterruptedException.class); + assertThat(interruptStatusPreserved) + .as("Interrupt status should be restored") + .isTrue(); + + // when the pending creation completes after the interrupt, the producer must be closed so that + // the exclusive producer doesn't leak + verify(producer, never()).closeAsync(); + producerFuture.complete(producer); + verify(producer, times(1)).closeAsync(); + } + @Test public void testDLogConfiguration() throws URISyntaxException, IOException { // The config yml is seeded with a fake bookie config. From b42a2ecdc77d2e77e3c4ce8437107a721eda1601 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 5 Jun 2026 19:51:39 +0300 Subject: [PATCH 041/213] [improve][misc] Upgrade Jetty to 12.1.10 (#25943) (cherry picked from commit c3a490b65477dcc43b4cec5cca27b7a1c8512c14) --- .../server/src/assemble/LICENSE.bin.txt | 78 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 24 +++--- pom.xml | 2 +- 3 files changed, 52 insertions(+), 52 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index d326a94353366..f2cce7d0ca875 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -392,43 +392,43 @@ The Apache Software License, Version 2.0 - org.asynchttpclient-async-http-client-2.15.0.jar - org.asynchttpclient-async-http-client-netty-utils-2.15.0.jar * Jetty - - org.eclipse.jetty-jetty-alpn-client-12.1.9.jar - - org.eclipse.jetty-jetty-alpn-conscrypt-server-12.1.9.jar - - org.eclipse.jetty-jetty-alpn-server-12.1.9.jar - - org.eclipse.jetty-jetty-annotations-12.1.9.jar - - org.eclipse.jetty-jetty-client-12.1.9.jar - - org.eclipse.jetty-jetty-http-12.1.9.jar - - org.eclipse.jetty-jetty-io-12.1.9.jar - - org.eclipse.jetty-jetty-jndi-12.1.9.jar - - org.eclipse.jetty-jetty-plus-12.1.9.jar - - org.eclipse.jetty-jetty-security-12.1.9.jar - - org.eclipse.jetty-jetty-server-12.1.9.jar - - org.eclipse.jetty-jetty-session-12.1.9.jar - - org.eclipse.jetty-jetty-util-12.1.9.jar - - org.eclipse.jetty-jetty-xml-12.1.9.jar - - org.eclipse.jetty.compression-jetty-compression-common-12.1.9.jar - - org.eclipse.jetty.compression-jetty-compression-gzip-12.1.9.jar - - org.eclipse.jetty.compression-jetty-compression-server-12.1.9.jar - - org.eclipse.jetty.ee-jetty-ee-webapp-12.1.9.jar - - org.eclipse.jetty.ee8-jetty-ee8-annotations-12.1.9.jar - - org.eclipse.jetty.ee8-jetty-ee8-nested-12.1.9.jar - - org.eclipse.jetty.ee8-jetty-ee8-plus-12.1.9.jar - - org.eclipse.jetty.ee8-jetty-ee8-proxy-12.1.9.jar - - org.eclipse.jetty.ee8-jetty-ee8-security-12.1.9.jar - - org.eclipse.jetty.ee8-jetty-ee8-servlet-12.1.9.jar - - org.eclipse.jetty.ee8-jetty-ee8-servlets-12.1.9.jar - - org.eclipse.jetty.ee8-jetty-ee8-webapp-12.1.9.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-api-12.1.9.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-common-12.1.9.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-server-12.1.9.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-servlet-12.1.9.jar + - org.eclipse.jetty-jetty-alpn-client-12.1.10.jar + - org.eclipse.jetty-jetty-alpn-conscrypt-server-12.1.10.jar + - org.eclipse.jetty-jetty-alpn-server-12.1.10.jar + - org.eclipse.jetty-jetty-annotations-12.1.10.jar + - org.eclipse.jetty-jetty-client-12.1.10.jar + - org.eclipse.jetty-jetty-http-12.1.10.jar + - org.eclipse.jetty-jetty-io-12.1.10.jar + - org.eclipse.jetty-jetty-jndi-12.1.10.jar + - org.eclipse.jetty-jetty-plus-12.1.10.jar + - org.eclipse.jetty-jetty-security-12.1.10.jar + - org.eclipse.jetty-jetty-server-12.1.10.jar + - org.eclipse.jetty-jetty-session-12.1.10.jar + - org.eclipse.jetty-jetty-util-12.1.10.jar + - org.eclipse.jetty-jetty-xml-12.1.10.jar + - org.eclipse.jetty.compression-jetty-compression-common-12.1.10.jar + - org.eclipse.jetty.compression-jetty-compression-gzip-12.1.10.jar + - org.eclipse.jetty.compression-jetty-compression-server-12.1.10.jar + - org.eclipse.jetty.ee-jetty-ee-webapp-12.1.10.jar + - org.eclipse.jetty.ee8-jetty-ee8-annotations-12.1.10.jar + - org.eclipse.jetty.ee8-jetty-ee8-nested-12.1.10.jar + - org.eclipse.jetty.ee8-jetty-ee8-plus-12.1.10.jar + - org.eclipse.jetty.ee8-jetty-ee8-proxy-12.1.10.jar + - org.eclipse.jetty.ee8-jetty-ee8-security-12.1.10.jar + - org.eclipse.jetty.ee8-jetty-ee8-servlet-12.1.10.jar + - org.eclipse.jetty.ee8-jetty-ee8-servlets-12.1.10.jar + - org.eclipse.jetty.ee8-jetty-ee8-webapp-12.1.10.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-api-12.1.10.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-common-12.1.10.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-server-12.1.10.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-servlet-12.1.10.jar - org.eclipse.jetty.toolchain-jetty-servlet-api-4.0.9.jar - - org.eclipse.jetty.websocket-jetty-websocket-core-client-12.1.9.jar - - org.eclipse.jetty.websocket-jetty-websocket-core-common-12.1.9.jar - - org.eclipse.jetty.websocket-jetty-websocket-core-server-12.1.9.jar - - org.eclipse.jetty.websocket-jetty-websocket-jetty-api-12.1.9.jar - - org.eclipse.jetty.websocket-jetty-websocket-jetty-client-12.1.9.jar - - org.eclipse.jetty.websocket-jetty-websocket-jetty-common-12.1.9.jar + - org.eclipse.jetty.websocket-jetty-websocket-core-client-12.1.10.jar + - org.eclipse.jetty.websocket-jetty-websocket-core-common-12.1.10.jar + - org.eclipse.jetty.websocket-jetty-websocket-core-server-12.1.10.jar + - org.eclipse.jetty.websocket-jetty-websocket-jetty-api-12.1.10.jar + - org.eclipse.jetty.websocket-jetty-websocket-jetty-client-12.1.10.jar + - org.eclipse.jetty.websocket-jetty-websocket-jetty-common-12.1.10.jar * SnakeYaml -- org.yaml-snakeyaml-2.0.jar * RocksDB - org.rocksdb-rocksdbjni-7.9.2.jar * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.45.0.jar @@ -568,9 +568,9 @@ BSD 3-clause "New" or "Revised" License * JLine -- jline-jline-2.14.6.jar -- ../licenses/LICENSE-JLine.txt * JLine3 -- org.jline-jline-3.21.0.jar -- ../licenses/LICENSE-JLine.txt * OW2 ASM - - org.ow2.asm-asm-9.9.1.jar -- ../licenses/LICENSE-ASM.txt - - org.ow2.asm-asm-commons-9.9.1.jar -- ../licenses/LICENSE-ASM.txt - - org.ow2.asm-asm-tree-9.9.1.jar -- ../licenses/LICENSE-ASM.txt + - org.ow2.asm-asm-9.10.jar -- ../licenses/LICENSE-ASM.txt + - org.ow2.asm-asm-commons-9.10.jar -- ../licenses/LICENSE-ASM.txt + - org.ow2.asm-asm-tree-9.10.jar -- ../licenses/LICENSE-ASM.txt BSD 2-Clause License * HdrHistogram -- org.hdrhistogram-HdrHistogram-2.1.9.jar -- ../licenses/LICENSE-HdrHistogram.txt diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index a51b2161e9b3a..efad8c79c253f 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -401,18 +401,18 @@ The Apache Software License, Version 2.0 - async-http-client-2.15.0.jar - async-http-client-netty-utils-2.15.0.jar * Jetty - - jetty-alpn-client-12.1.9.jar - - jetty-client-12.1.9.jar - - jetty-compression-common-12.1.9.jar - - jetty-compression-gzip-12.1.9.jar - - jetty-http-12.1.9.jar - - jetty-io-12.1.9.jar - - jetty-util-12.1.9.jar - - jetty-websocket-core-client-12.1.9.jar - - jetty-websocket-core-common-12.1.9.jar - - jetty-websocket-jetty-api-12.1.9.jar - - jetty-websocket-jetty-client-12.1.9.jar - - jetty-websocket-jetty-common-12.1.9.jar + - jetty-alpn-client-12.1.10.jar + - jetty-client-12.1.10.jar + - jetty-compression-common-12.1.10.jar + - jetty-compression-gzip-12.1.10.jar + - jetty-http-12.1.10.jar + - jetty-io-12.1.10.jar + - jetty-util-12.1.10.jar + - jetty-websocket-core-client-12.1.10.jar + - jetty-websocket-core-common-12.1.10.jar + - jetty-websocket-jetty-api-12.1.10.jar + - jetty-websocket-jetty-client-12.1.10.jar + - jetty-websocket-jetty-common-12.1.10.jar * SnakeYaml -- snakeyaml-2.0.jar * Google Error Prone Annotations - error_prone_annotations-2.45.0.jar * Javassist -- javassist-3.25.0-GA.jar diff --git a/pom.xml b/pom.xml index 1c364712d7dff..623bcacc14c58 100644 --- a/pom.xml +++ b/pom.xml @@ -189,7 +189,7 @@ flexible messaging model and an intuitive client API. 5.7.1 4.1.135.Final 0.0.26.Final - 12.1.9 + 12.1.10 9.4.58.v20250814 2.5.2 From bba285f12bff7dc602b461bd2e46b8a404f7b1db Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 27 Apr 2026 14:43:00 +0800 Subject: [PATCH 042/213] [feat][pip] PIP-469: Legacy-aware topic policies backend routing and metadata-store topic policies (#25547) (cherry picked from commit 33fe7559b4d1a2cfb9d35756d38f7bad72b54309) --- pip/pip-469.md | 270 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 270 insertions(+) create mode 100644 pip/pip-469.md diff --git a/pip/pip-469.md b/pip/pip-469.md new file mode 100644 index 0000000000000..69036381f0f01 --- /dev/null +++ b/pip/pip-469.md @@ -0,0 +1,270 @@ +# PIP-469: Legacy-aware topic policies backend routing and metadata-store topic policies + +# Background knowledge + +Apache Pulsar introduced topic-level policies in [PIP-39](pip-39.md). A broker reads and writes these policies through +`TopicPoliciesService`. The default implementation, +`SystemTopicBasedTopicPoliciesService`, persists topic policy changes in the namespace `__change_events` system topic and +keeps an in-memory cache on brokers that own bundles for that namespace. + +[PIP-92](pip-92.md) extended topic policies with the distinction between local and global policies. Any +`TopicPoliciesService` implementation therefore needs to handle two independent values for the same topic: the +cluster-local policy state and the globally visible policy state. + +[PIP-376](pip-376.md) made `TopicPoliciesService` pluggable through the broker configuration +`topicPoliciesServiceClassName`. That change removed the hard coupling between topic policies and system topics, but the +backend choice is still broker-wide. During upgrade from the default system-topic backend, brokers still need a way to +recognize namespaces that already have topic-policies state in `__change_events`, so those namespaces do not silently +move to another backend. + +# Motivation + +The system-topic-based topic policies implementation works by appending topic policy changes to a `__change_events` +topic in each namespace. It works well when this topic has already been loaded by a broker, then all topic policies +operations just access the in-memory cache. However, in cold start scenarios, for example when the owner broker is down +during a restart, the new owner broker has to create a reader on the `__change_events` topic and wait for it to catch +up before it can read any topic policies, which is required in the path of loading a topic in the same namespace. This +adds significant latency to the topic load path, especially before the topic is compacted. + +Things become worse when many `__change_events` topics move to a restarting broker. The new owner broker has to create +many readers and replay all messages on these topics. This leads to high pressure on BookKeeper and can cause +`Too many requests on the same bookie` errors in `GetLastMessageId` RPCs. + +A metadata-store-backed topic policies backend is attractive because it removes the extra lifecycle and operational +dependency of a dedicated `__change_events` topic. A metadata-cache-based implementation can still provide caching and +change notifications, while avoiding the cold-start latency of waiting for a system-topic reader to initialize and +catch up. + +There is a second operational requirement: operators need a safe gradual rollout path. Existing namespaces that already +have topic-policies state in `__change_events` must stay on the system-topic backend, while newly created namespaces +should be able to use the broker-configured backend. This does not require a new namespace policy. For the upgrade case +from the default configuration, the existence of `__change_events` is already a conservative legacy marker. + +# Goals + +## In Scope + +- Add a metadata-store-backed `TopicPoliciesService` implementation that does not depend on system topics. +- Add routing logic that forces the system-topic backend for namespaces that already have `__change_events`. +- Keep using the broker-level `topicPoliciesServiceClassName` for namespaces that do not have `__change_events`, + including newly created namespaces. + +## Out of Scope + +- Adding a migration framework that moves topic policies data between backends automatically. + +# High Level Design + +When topic-level policies are enabled, the broker instantiates a `LegacyAwareTopicPoliciesService` instead of using the +configured implementation directly. + +The wrapper always has access to two backends: + +- `SystemTopicBasedTopicPoliciesService` +- The broker-configured `topicPoliciesServiceClassName` + +For each namespace, the wrapper checks whether the topic-policies system topic `persistent://{tenant}/{namespace}/__change_events` +already exists: + +- If it exists, the namespace is treated as a legacy system-topic namespace and all topic-policies operations are + routed to `SystemTopicBasedTopicPoliciesService`. +- If it does not exist, the namespace uses the broker-configured `topicPoliciesServiceClassName`. + +This rule is intentionally conservative. If `__change_events` exists, the broker assumes that namespace may already +contain topic-policies state in the system-topic backend and therefore must not be moved implicitly. + +This proposal also introduces `MetadataStoreTopicPoliciesService`, a concrete `TopicPoliciesService` implementation +that stores topic policies in dedicated metadata-store paths: + +- Global topic policies are stored in the configuration metadata store. +- Local topic policies are stored in the local metadata store. + +This keeps the storage scope aligned with the semantics introduced by PIP-92 and avoids writing topic policies through +managed-ledger metadata side effects. + +# Detailed Design + +## Design & Implementation Details + +### Startup and validation + +`PulsarService#initTopicPoliciesService()` continues to respect `topicLevelPoliciesEnabled`. When topic-level policies +are disabled, behavior is unchanged and `TopicPoliciesService.DISABLED` is used. + +When topic-level policies are enabled, the broker constructs: + +```java +new LegacyAwareTopicPoliciesService( + this, + new SystemTopicBasedTopicPoliciesService(this), + configuredTopicPoliciesService) +``` + +Broker startup validates both backends: + +- `SystemTopicBasedTopicPoliciesService` must be instantiable. +- The configured `topicPoliciesServiceClassName` must be instantiable. + +If either backend cannot be instantiated or started, broker startup fails. There is no per-request fallback from one +backend to another. + +### Namespace-scoped service routing + +`LegacyAwareTopicPoliciesService` is responsible for: + +- Checking whether `__change_events` exists for the namespace by using + `NamespaceEventsSystemTopicFactory.checkSystemTopicExists(namespace, EventType.TOPIC_POLICY, pulsarService)`. +- Routing `getTopicPoliciesAsync`, `updateTopicPoliciesAsync`, `deleteTopicPoliciesAsync`, and listener operations to + the system-topic backend when the system topic exists. +- Routing the same operations to the configured backend when the system topic does not exist. + +The system-topic existence check can be cached per namespace in memory, but the routing rule is defined by actual topic +existence rather than by new namespace metadata. + +This means: + +- Existing namespaces that already materialized `__change_events` continue to use the system-topic backend. +- Namespaces that never created `__change_events` use the broker-configured backend. +- Newly created namespaces use the broker-configured backend because `__change_events` does not exist yet. + +If `__change_events` is later deleted, the namespace falls back to the broker-configured backend on subsequent +resolution. This matches current system-topic behavior, which already treats a missing `__change_events` topic as +meaning the system-topic-backed topic-policies state is gone. + +### Metadata-backed topic policies service + +`MetadataStoreTopicPoliciesService` implements `TopicPoliciesService` with the following storage model: + +- Topic names are normalized to the partitioned topic name, so all partitions share the same topic-policies record. +- Global policies are stored in the configuration metadata store path: + `/admin/topic-policies/{tenant}/{namespace}/{domain}/{encodedTopic}`. +- Local policies are stored in the local metadata store path: + `/admin/local-policies/topic-policies/{tenant}/{namespace}/{domain}/{encodedTopic}`. + +Each node stores a serialized `TopicPolicies` document. The backend writes and reads the two scopes independently: + +- Reads with `GetType.GLOBAL_ONLY` only touch the global path and return a `TopicPolicies` object whose `isGlobal` + flag is `true`. +- Reads with `GetType.LOCAL_ONLY` only touch the local path and return a `TopicPolicies` object whose `isGlobal` flag + is `false`. +- Updates with `isGlobalPolicy=true` only modify the global path. +- Updates with `isGlobalPolicy=false` only modify the local path. + +Deletes remove the local record and, unless `keepGlobalPoliciesAfterDeleting` is set, also remove the global record. +This matches the existing `TopicPoliciesService` deletion contract. + +This design intentionally uses dedicated metadata nodes instead of piggybacking on `PartitionedTopicMetadata` or +`ManagedLedgerInfo`. That keeps local/global visibility correct and avoids losing topic policies during normal +managed-ledger metadata updates. + +### Listener behavior + +The backend registers watchers on both metadata stores: + +- A change on the local path re-reads the local node and notifies listeners with the latest local `TopicPolicies` or + `null` if the local node was removed. +- A change on the global path re-reads the global node and notifies listeners with the latest global `TopicPolicies` + or `null` if the global node was removed. + +This preserves runtime updates for already loaded topics, including global topic policies. The backend does not add an +append-only replay log; it relies on metadata-store notifications and read-after-notify refresh. + +## Public-facing Changes + +### Public API + +No new namespace policy field is introduced. + +No new namespace admin REST endpoint or Java admin client method is introduced. + +Changing the topic-policies backend for a namespace is not a public operation in this proposal. The routing rule is +derived from `__change_events` existence plus the broker-level configuration. + +### Binary protocol + +No binary protocol changes. + +### Configuration + +- `topicPoliciesServiceClassName` + - Continues to define the broker-configured `TopicPoliciesService` implementation. + - Namespaces that do not have `__change_events` use this backend. + - Namespaces that already have `__change_events` keep using `SystemTopicBasedTopicPoliciesService` regardless of + this value. + +### CLI + +No CLI change in this proposal. + +### Metrics + +No new metric is required. + +# Backward & Forward Compatibility + +## Upgrade + +The intended upgrade flow is: + +1. Upgrade brokers to a version that understands legacy-aware backend routing. +2. Change `topicPoliciesServiceClassName` to the alternate backend if newly created namespaces should use it. +3. Existing namespaces that already have `__change_events` continue to use `SystemTopicBasedTopicPoliciesService`. +4. Namespaces that do not have `__change_events`, including newly created namespaces, use the configured backend. + +No namespace metadata backfill is required. + +This upgrade rule is intentionally conservative: + +- If `__change_events` exists, the namespace stays on the system-topic backend. +- If `__change_events` does not exist, the namespace uses the configured backend. + +This means some namespaces with an empty but already-created `__change_events` topic may continue using the +system-topic backend. That is acceptable because it avoids missing legacy state. + +## Downgrade / Rollback + +Rolling back to a broker version that does not understand legacy-aware routing returns topic-policies backend +selection to pure broker-wide behavior. + +- The older broker will no longer special-case namespaces that have `__change_events`. +- Operators will need to choose one broker-wide backend for the rollback cluster, or migrate data before rollback if + both legacy system-topic namespaces and metadata-store namespaces must coexist. + +## Pulsar Geo-Replication Upgrade & Downgrade/Rollback Considerations + +This proposal does not introduce a new geo-replication protocol for topic policies. + +- Global topic policies stay in the configuration metadata store and therefore keep global visibility semantics. +- Local topic policies stay in the local metadata store and therefore keep cluster-local visibility semantics. +- Legacy namespaces are recognized by the existence of `__change_events`, which is already shared broker-visible topic + metadata. + +# Alternatives + +## Keep a single broker-wide topic policies backend + +This keeps the implementation simpler, but it does not solve the operational requirement to keep existing namespaces on +their current backend while directing newly created namespaces to a different one. + +## Persist an explicit namespace backend marker + +This would also solve the upgrade problem, but it introduces new namespace-scoped metadata changes that are not +necessary for the default-system-topic upgrade path. The proposal prefers to reuse the already existing +`__change_events` artifact as the legacy marker. + +## Add a user-managed namespace override API + +This provides more flexibility than needed, but it also reintroduces runtime switching, rollback ambiguity, and the +risk of one namespace being served by different backends if brokers do not resolve the override identically. The +proposal intentionally avoids this surface. + +# General Notes + +This proposal is a follow-up to [PIP-376](pip-376.md). It keeps backend selection pluggable, but handles upgrade from +the legacy system-topic backend by reusing `__change_events` as the compatibility marker instead of introducing a new +namespace-level policy or namespace-level metadata field. + +# Links + +* Mailing List discussion thread: https://lists.apache.org/thread/sn2pyyl9p1vm5vr8j8qssxbbksm2bzfr +* Mailing List voting thread: https://lists.apache.org/thread/b5mfqrmxcwwzjbkhzv6t6t12gtvjz1so From a5e33ec8402cae25572f53f7d2fa6c2c6e5be033 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 18 May 2026 19:16:47 +0800 Subject: [PATCH 043/213] [feat][broker] PIP-469: Legacy-aware topic policies backend routing and metadata-store topic policies (#25707) (cherry picked from commit 8652efa4d60b791a5f1ee4e52f7ffda6ebbbb256) (cherry picked from commit 712372f9ee18ae5d5bd8e35ca6ef45473c0a3bb5) --- pip/pip-469.md | 37 ++- .../pulsar/broker/ServiceConfiguration.java | 12 +- .../apache/pulsar/broker/PulsarService.java | 10 +- .../pulsar/broker/service/AbstractTopic.java | 2 +- .../LegacyAwareTopicPoliciesService.java | 142 +++++++++ .../MetadataStoreTopicPoliciesService.java | 277 ++++++++++++++++++ .../SystemTopicBasedTopicPoliciesService.java | 2 +- .../broker/service/TopicPoliciesService.java | 15 +- .../service/persistent/PersistentTopic.java | 8 +- .../admin/MetadataStoreTopicPoliciesTest.java | 72 +++++ .../broker/admin/TopicPoliciesTest.java | 112 +++++-- .../LegacyAwareTopicPoliciesServiceTest.java | 190 ++++++++++++ ...temTopicBasedTopicPoliciesServiceTest.java | 4 +- .../broker/service/TopicPolicyTestUtils.java | 7 + 14 files changed, 846 insertions(+), 44 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/service/LegacyAwareTopicPoliciesService.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/service/MetadataStoreTopicPoliciesService.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/MetadataStoreTopicPoliciesTest.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/LegacyAwareTopicPoliciesServiceTest.java diff --git a/pip/pip-469.md b/pip/pip-469.md index 69036381f0f01..4734adc127d4c 100644 --- a/pip/pip-469.md +++ b/pip/pip-469.md @@ -105,8 +105,15 @@ Broker startup validates both backends: - `SystemTopicBasedTopicPoliciesService` must be instantiable. - The configured `topicPoliciesServiceClassName` must be instantiable. -If either backend cannot be instantiated or started, broker startup fails. There is no per-request fallback from one -backend to another. +`LegacyAwareTopicPoliciesService#start` starts only the configured backend. It intentionally does not call +`SystemTopicBasedTopicPoliciesService#start`, because that start path registers a namespace-bundle ownership listener +whose only purpose is to eagerly create a reader on `/__change_events` when a namespace bundle is loaded. +Under legacy-aware routing, that eager optimization would be counterproductive because it can create readers for +namespaces that do not have topic policies in `__change_events`. For legacy namespaces, the system-topic reader and +policy cache are initialized lazily by the routed system-topic backend operations. + +If either backend cannot be instantiated, or if the configured backend cannot be started, broker startup fails. There is +no per-request fallback from one backend to another. ### Namespace-scoped service routing @@ -118,6 +125,10 @@ backend to another. the system-topic backend when the system topic exists. - Routing the same operations to the configured backend when the system topic does not exist. +Listener registration is routed through `TopicPoliciesService#registerListenerAsync`. This lets the wrapper resolve the +namespace backend before registering the listener, and the listener is registered only on the selected backend instead +of being registered on both backends. + The system-topic existence check can be cached per namespace in memory, but the routing rule is defined by actual topic existence rather than by new namespace metadata. @@ -137,9 +148,13 @@ meaning the system-topic-backed topic-policies state is gone. - Topic names are normalized to the partitioned topic name, so all partitions share the same topic-policies record. - Global policies are stored in the configuration metadata store path: - `/admin/topic-policies/{tenant}/{namespace}/{domain}/{encodedTopic}`. + `/admin/topic-policies/global/{tenant}/{namespace}/{domain}/{encodedTopic}`. - Local policies are stored in the local metadata store path: - `/admin/local-policies/topic-policies/{tenant}/{namespace}/{domain}/{encodedTopic}`. + `/admin/topic-policies/local/{tenant}/{namespace}/{domain}/{encodedTopic}`. + +To avoid possible conflicts like the listener registered on the `/admin/local-policies` path from +`BrokerService#handleMetadataChanges`, these two paths share the same root path `/admin/topic-policies`, which is not +used by any other component. Each node stores a serialized `TopicPolicies` document. The backend writes and reads the two scopes independently: @@ -159,6 +174,11 @@ managed-ledger metadata updates. ### Listener behavior +`TopicPoliciesService` adds `registerListenerAsync(TopicName, TopicPolicyListener)` for listener registration. The +existing synchronous `registerListener(TopicName, TopicPolicyListener)` method is retained as a deprecated compatibility +hook for existing custom implementations, and the default async method delegates to it. Implementations that need async +routing or initialization, such as `LegacyAwareTopicPoliciesService`, override `registerListenerAsync` directly. + The backend registers watchers on both metadata stores: - A change on the local path re-reads the local node and notifies listeners with the latest local `TopicPolicies` or @@ -173,6 +193,11 @@ append-only replay log; it relies on metadata-store notifications and read-after ### Public API +The `TopicPoliciesService` extension point gains a default +`CompletableFuture registerListenerAsync(TopicName, TopicPolicyListener)` method. Existing implementations +remain compatible because `registerListener(TopicName, TopicPolicyListener)` is retained and used by the default async +implementation. + No new namespace policy field is introduced. No new namespace admin REST endpoint or Java admin client method is introduced. @@ -221,6 +246,10 @@ This upgrade rule is intentionally conservative: This means some namespaces with an empty but already-created `__change_events` topic may continue using the system-topic backend. That is acceptable because it avoids missing legacy state. +Existing custom `TopicPoliciesService` implementations that only implement the synchronous `registerListener` method +continue to work through the default `registerListenerAsync` bridge. Implementations can override +`registerListenerAsync` when registration itself needs asynchronous backend resolution or initialization. + ## Downgrade / Rollback Rolling back to a broker version that does not understand legacy-aware routing returns topic-policies backend diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 9df20329db06a..132dfeb1358bf 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -1731,8 +1731,16 @@ The max allowed delay for delayed delivery (in milliseconds). If the broker rece @FieldContext( category = CATEGORY_SERVER, - doc = "The class name of the topic policies service. The default config only takes affect when the " - + "systemTopicEnable config is true" + doc = """ + The class name of the topic policies service. There are 2 built-in implementations: + 1. "org.apache.pulsar.broker.service.SystemTopicBasedTopicPoliciesService" (default) + It stores a topic's policies in the `__change_events` topic. If `systemTopicEnabled` is false, + the topic policies will just be disabled + 2. "org.apache.pulsar.broker.service.MetadataStoreTopicPoliciesService" + It stores a topic's policies in the metadata store. If `systemTopicEnabled` is true and the + topic's namespace has a `__change_events` topic, the policies will still be stored in the + `__change_events` topic for backward compatibility. + """ ) private String topicPoliciesServiceClassName = "org.apache.pulsar.broker.service.SystemTopicBasedTopicPoliciesService"; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index a77e09d79cad3..e74bc51a5329c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -104,6 +104,7 @@ import org.apache.pulsar.broker.rest.Topics; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.HealthChecker; +import org.apache.pulsar.broker.service.LegacyAwareTopicPoliciesService; import org.apache.pulsar.broker.service.PulsarMetadataEventSynchronizer; import org.apache.pulsar.broker.service.SystemTopicBasedTopicPoliciesService; import org.apache.pulsar.broker.service.Topic; @@ -2232,8 +2233,15 @@ private TopicPoliciesService initTopicPoliciesService() throws Exception { return TopicPoliciesService.DISABLED; } } - return (TopicPoliciesService) Reflections.createInstance(className, + final var configuredService = (TopicPoliciesService) Reflections.createInstance(className, Thread.currentThread().getContextClassLoader()); + if (!config.isSystemTopicEnabled()) { + LOG.info("[{}] System topic is disabled, using configured topic policies service without legacy routing", + className); + return configuredService; + } + return new LegacyAwareTopicPoliciesService(this, new SystemTopicBasedTopicPoliciesService(this), + configuredService); } /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 648d97992aacc..988ebe50003b7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -631,7 +631,7 @@ protected boolean isProducersExceeded(boolean isRemote) { protected void registerTopicPolicyListener() { brokerService.getPulsar().getTopicPoliciesService() - .registerListener(TopicName.getPartitionedTopicName(topic), this); + .registerListenerAsync(TopicName.getPartitionedTopicName(topic), this); } protected void unregisterTopicPolicyListener() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/LegacyAwareTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/LegacyAwareTopicPoliciesService.java new file mode 100644 index 0000000000000..6e1bfd7bacc42 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/LegacyAwareTopicPoliciesService.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import com.github.benmanes.caffeine.cache.AsyncCacheLoader; +import com.github.benmanes.caffeine.cache.AsyncLoadingCache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.google.common.annotations.VisibleForTesting; +import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.function.Consumer; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory; +import org.apache.pulsar.common.events.EventType; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.TopicPolicies; +import org.jspecify.annotations.NonNull; + +/** + * Routes topic policy operations to the legacy system-topic backend when a namespace already has + * a topic-policy {@code __change_events} system topic, and otherwise to the configured backend. + */ +public class LegacyAwareTopicPoliciesService implements TopicPoliciesService { + + private final AsyncLoadingCache isLegacyNamespace; + @VisibleForTesting + final SystemTopicBasedTopicPoliciesService systemTopicService; + private final TopicPoliciesService configuredService; + + public LegacyAwareTopicPoliciesService(PulsarService pulsar, + SystemTopicBasedTopicPoliciesService systemTopicService, + TopicPoliciesService configuredService) { + // Generally, we only need to check if the __change_events topic exists once because the __change_events topic + // should only be created by broker before the upgrade, where `SystemTopicBasedTopicPoliciesService` is + // configured as the topic policies service. + this.isLegacyNamespace = Caffeine.newBuilder().expireAfterWrite(Duration.ofHours(1)) + .buildAsync(new AsyncCacheLoader<>() { + @NonNull + @Override + public CompletableFuture asyncLoad(NamespaceName key, + @NonNull Executor executor) { + return NamespaceEventsSystemTopicFactory.checkSystemTopicExists(key, EventType.TOPIC_POLICY, + pulsar); + } + }); + this.systemTopicService = systemTopicService; + this.configuredService = configuredService; + if (configuredService instanceof SystemTopicBasedTopicPoliciesService) { + throw new IllegalArgumentException( + "configuredService should not be an instance of SystemTopicBasedTopicPoliciesService"); + } + } + + @Override + public void start(PulsarService pulsarService) { + // We should not call `systemTopicService.start()`, which just registers a namespace bundle listener to create + // a reader on `/__change_events` when the namespace's bundle is loaded firstly. It's just an + // optimization to create the reader before loading any topic. However, it could create a reader on a namespace + // that does not even have the __change_events topic. + configuredService.start(pulsarService); + } + + @Override + public void close() throws Exception { + try { + configuredService.close(); + } finally { + systemTopicService.close(); + } + } + + @Override + public CompletableFuture> getTopicPoliciesAsync(TopicName topicName, GetType type) { + return resolveService(topicName.getNamespaceObject()) + .thenCompose(service -> service.getTopicPoliciesAsync(topicName, type)); + } + + @Override + public CompletableFuture updateTopicPoliciesAsync(TopicName topicName, boolean isGlobalPolicy, + boolean skipUpdateWhenTopicPolicyDoesntExist, + Consumer policyUpdater) { + return resolveService(topicName.getNamespaceObject()) + .thenCompose(service -> service.updateTopicPoliciesAsync(topicName, isGlobalPolicy, + skipUpdateWhenTopicPolicyDoesntExist, policyUpdater)); + } + + @Override + public CompletableFuture deleteTopicPoliciesAsync(TopicName topicName) { + return resolveService(topicName.getNamespaceObject()) + .thenCompose(service -> service.deleteTopicPoliciesAsync(topicName)); + } + + @Override + public CompletableFuture deleteTopicPoliciesAsync(TopicName topicName, + boolean keepGlobalPoliciesAfterDeleting) { + return resolveService(topicName.getNamespaceObject()) + .thenCompose(service -> service.deleteTopicPoliciesAsync(topicName, + keepGlobalPoliciesAfterDeleting)); + } + + @Override + public CompletableFuture registerListenerAsync(TopicName topicName, TopicPolicyListener listener) { + return resolveService(topicName.getNamespaceObject()) + .thenCompose(service -> service.registerListenerAsync(topicName, listener)); + } + + @Override + public boolean registerListener(TopicName topicName, TopicPolicyListener listener) { + throw new RuntimeException("should not be called"); + } + + @Override + public void unregisterListener(TopicName topicName, TopicPolicyListener listener) { + configuredService.unregisterListener(topicName, listener); + systemTopicService.unregisterListener(topicName, listener); + } + + @VisibleForTesting + CompletableFuture resolveService(NamespaceName namespace) { + return isLegacyNamespace.get(namespace) + .thenApply(isLegacy -> isLegacy ? systemTopicService : configuredService); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/MetadataStoreTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/MetadataStoreTopicPoliciesService.java new file mode 100644 index 0000000000000..218c37472a3d5 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/MetadataStoreTopicPoliciesService.java @@ -0,0 +1,277 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import com.google.common.annotations.VisibleForTesting; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.namespace.NamespaceService; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.TopicPolicies; +import org.apache.pulsar.common.util.Codec; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.metadata.api.MetadataCache; +import org.apache.pulsar.metadata.api.MetadataStore; +import org.apache.pulsar.metadata.api.MetadataStoreException.NotFoundException; +import org.apache.pulsar.metadata.api.Notification; +import org.apache.pulsar.metadata.api.NotificationType; +import org.jspecify.annotations.Nullable; + +/** + * Topic policies service backed by Pulsar metadata stores. + */ +@Slf4j +public class MetadataStoreTopicPoliciesService implements TopicPoliciesService { + + public static final String GLOBAL_POLICIES_ROOT = "/admin/topic-policies/global"; + public static final String LOCAL_POLICIES_ROOT = "/admin/topic-policies/local"; + + private final AtomicBoolean closed = new AtomicBoolean(false); + private final Map> listeners = new ConcurrentHashMap<>(); + private MetadataCache localPoliciesCache; + private MetadataCache globalPoliciesCache; + + @Override + public void start(PulsarService pulsar) { + MetadataStore localStore = pulsar.getLocalMetadataStore(); + MetadataStore configurationStore = pulsar.getConfigurationMetadataStore(); + this.localPoliciesCache = localStore.getMetadataCache(TopicPolicies.class); + this.globalPoliciesCache = configurationStore.getMetadataCache(TopicPolicies.class); + localStore.registerListener(notification -> handleNotification(notification, false)); + configurationStore.registerListener(notification -> handleNotification(notification, true)); + } + + @Override + public CompletableFuture deleteTopicPoliciesAsync(TopicName topicName) { + return deleteTopicPoliciesAsync(topicName, false); + } + + @Override + public CompletableFuture deleteTopicPoliciesAsync(TopicName topicName, + boolean keepGlobalPoliciesAfterDeleting) { + TopicName partitionedTopicName = normalizeTopicName(topicName); + if (NamespaceService.isHeartbeatNamespace(partitionedTopicName.getNamespaceObject())) { + return CompletableFuture.completedFuture(null); + } + if (closed.get()) { + return CompletableFuture.failedFuture(new BrokerServiceException(getClass().getName() + " is closed.")); + } + CompletableFuture deleteLocal = + deleteIfExists(localPoliciesCache, pathFor(partitionedTopicName, false)); + if (keepGlobalPoliciesAfterDeleting) { + return deleteLocal; + } + CompletableFuture deleteGlobal = + deleteIfExists(globalPoliciesCache, pathFor(partitionedTopicName, true)); + return CompletableFuture.allOf(deleteLocal, deleteGlobal); + } + + @Override + public CompletableFuture updateTopicPoliciesAsync(TopicName topicName, boolean isGlobalPolicy, + boolean skipUpdateWhenTopicPolicyDoesntExist, + Consumer policyUpdater) { + TopicName partitionedTopicName = normalizeTopicName(topicName); + if (NamespaceService.isHeartbeatNamespace(partitionedTopicName.getNamespaceObject())) { + return CompletableFuture.failedFuture(new BrokerServiceException.NotAllowedException( + "Not allowed to update topic policy for the heartbeat topic")); + } + if (closed.get()) { + return CompletableFuture.failedFuture(new BrokerServiceException(getClass().getName() + " is closed.")); + } + MetadataCache cache = cache(isGlobalPolicy); + String path = pathFor(partitionedTopicName, isGlobalPolicy); + CompletableFuture updateFuture; + if (skipUpdateWhenTopicPolicyDoesntExist) { + updateFuture = cache.readModifyUpdate(path, + current -> updatePolicies(Optional.of(current), isGlobalPolicy, policyUpdater)); + } else { + updateFuture = cache.readModifyUpdateOrCreate(path, + current -> updatePolicies(current, isGlobalPolicy, policyUpdater)); + } + return updateFuture.thenAccept(__ -> { }).exceptionally(error -> { + if (skipUpdateWhenTopicPolicyDoesntExist + && FutureUtil.unwrapCompletionException(error) instanceof NotFoundException) { + return null; + } + throw FutureUtil.wrapToCompletionException(error); + }); + } + + @Override + public CompletableFuture> getTopicPoliciesAsync(TopicName topicName, GetType type) { + TopicName partitionedTopicName = normalizeTopicName(topicName); + if (NamespaceService.isHeartbeatNamespace(partitionedTopicName.getNamespaceObject())) { + return CompletableFuture.completedFuture(Optional.empty()); + } + if (closed.get()) { + return CompletableFuture.completedFuture(Optional.empty()); + } + boolean global = type == GetType.GLOBAL_ONLY; + return cache(global).get(pathFor(partitionedTopicName, global)) + .thenApply(policies -> policies.map(policy -> cloneWithScope(policy, global))); + } + + @Override + public boolean registerListener(TopicName topicName, TopicPolicyListener listener) { + listeners.compute(normalizeTopicName(topicName), (__, topicListeners) -> { + if (topicListeners == null) { + topicListeners = new CopyOnWriteArrayList<>(); + } + topicListeners.add(listener); + return topicListeners; + }); + return true; + } + + @Override + public void unregisterListener(TopicName topicName, TopicPolicyListener listener) { + listeners.computeIfPresent(normalizeTopicName(topicName), (__, topicListeners) -> { + topicListeners.remove(listener); + return topicListeners.isEmpty() ? null : topicListeners; + }); + } + + @Override + public void close() { + if (closed.compareAndSet(false, true)) { + listeners.clear(); + if (localPoliciesCache != null) { + localPoliciesCache.invalidateAll(); + } + if (globalPoliciesCache != null) { + globalPoliciesCache.invalidateAll(); + } + } + } + + private MetadataCache cache(boolean isGlobalPolicy) { + return isGlobalPolicy ? globalPoliciesCache : localPoliciesCache; + } + + private CompletableFuture deleteIfExists(MetadataCache cache, String path) { + return cache.delete(path).handle((__, error) -> { + cache.invalidate(path); + if (error == null || FutureUtil.unwrapCompletionException(error) instanceof NotFoundException) { + return null; + } + throw FutureUtil.wrapToCompletionException(error); + }); + } + + private static TopicPolicies updatePolicies(Optional currentPolicies, + boolean isGlobalPolicy, + Consumer policyUpdater) { + TopicPolicies policies = currentPolicies.map(TopicPolicies::clone).orElseGet(TopicPolicies::new); + policies.setIsGlobal(isGlobalPolicy); + policyUpdater.accept(policies); + return policies; + } + + private void handleNotification(Notification notification, boolean isGlobalPolicy) { + if (closed.get() + || (notification.getType() != NotificationType.Created + && notification.getType() != NotificationType.Modified + && notification.getType() != NotificationType.Deleted)) { + return; + } + String path = notification.getPath(); + String root = isGlobalPolicy ? GLOBAL_POLICIES_ROOT : LOCAL_POLICIES_ROOT; + Optional topicName = topicNameFromPath(root, path); + if (topicName.isEmpty()) { + return; + } + MetadataCache cache = cache(isGlobalPolicy); + cache.invalidate(path); + if (notification.getType() == NotificationType.Deleted) { + notifyListeners(topicName.get(), null); + return; + } + cache.get(path).whenComplete((policies, error) -> { + if (error != null) { + log.warn("[{}] Failed to refresh topic policies after metadata notification", path, error); + return; + } + notifyListeners(topicName.get(), + policies.map(policy -> cloneWithScope(policy, isGlobalPolicy)).orElse(null)); + }); + } + + private void notifyListeners(TopicName topicName, @Nullable TopicPolicies policies) { + List topicListeners = listeners.get(topicName); + if (topicListeners == null) { + return; + } + for (TopicPolicyListener listener : topicListeners) { + try { + listener.onUpdate(policies == null ? null : policies.clone()); + } catch (Throwable error) { + log.error("[{}] Call topic policy listener error", topicName, error); + } + } + } + + private static TopicName normalizeTopicName(TopicName topicName) { + return TopicName.get(topicName.getPartitionedTopicName()); + } + + private static TopicPolicies cloneWithScope(TopicPolicies policies, boolean isGlobalPolicy) { + TopicPolicies cloned = policies.clone(); + cloned.setIsGlobal(isGlobalPolicy); + return cloned; + } + + @VisibleForTesting + public CompletableFuture> getTopicPoliciesDirectFromStore(TopicName topicName, + boolean isGlobal) { + String path = pathFor(topicName, isGlobal); + MetadataCache c = cache(isGlobal); + c.invalidate(path); + return c.get(path).thenApply(opt -> opt.map(p -> cloneWithScope(p, isGlobal))); + } + + @VisibleForTesting + static String pathFor(TopicName topicName, boolean isGlobalPolicy) { + TopicName partitionedTopicName = normalizeTopicName(topicName); + return (isGlobalPolicy ? GLOBAL_POLICIES_ROOT : LOCAL_POLICIES_ROOT) + + "/" + partitionedTopicName.getTenant() + + "/" + partitionedTopicName.getNamespacePortion() + + "/" + partitionedTopicName.getDomain() + + "/" + partitionedTopicName.getEncodedLocalName(); + } + + @VisibleForTesting + private static Optional topicNameFromPath(String root, String path) { + if (!path.startsWith(root + "/")) { + return Optional.empty(); + } + String[] parts = path.substring(root.length() + 1).split("/", 4); + if (parts.length != 4) { + return Optional.empty(); + } + return Optional.of(TopicName.get(parts[2], parts[0], parts[1], Codec.decode(parts[3]))); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index b056cedcdb87a..b47134f97b4e1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -654,7 +654,7 @@ protected CompletableFuture> createSystemT return systemTopicClient.newReaderAsync(); } - private void removeOwnedNamespaceBundleAsync(NamespaceBundle namespaceBundle) { + void removeOwnedNamespaceBundleAsync(NamespaceBundle namespaceBundle) { NamespaceName namespace = namespaceBundle.getNamespaceObject(); if (NamespaceService.isHeartbeatNamespace(namespace)) { return; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java index 239c1d3d9bad4..7f49686a17fcf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java @@ -100,6 +100,15 @@ default void start(PulsarService pulsar) { default void close() throws Exception { } + + /** + * @implNote This method is never called unless by the default implementation of + * {@link TopicPoliciesService#registerListenerAsync(TopicName, TopicPolicyListener)}, which is actually called + * internally. This method is only retained for backward compatibility on custom implementations. + */ + @Deprecated + boolean registerListener(TopicName topicName, TopicPolicyListener listener); + /** * Registers a listener for topic policies updates. * @@ -110,10 +119,10 @@ default void close() throws Exception { * guaranteed to be received by the listener. * In summary, the listener is guaranteed to receive only the latest value. *

- * - * @return true if the listener is registered successfully */ - boolean registerListener(TopicName topicName, TopicPolicyListener listener); + default CompletableFuture registerListenerAsync(TopicName topicName, TopicPolicyListener listener) { + return CompletableFuture.completedFuture(registerListener(topicName, listener)); + } /** * Unregister the topic policies listener. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 5d397706f43cc..30289ee50f5a3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -4779,7 +4779,10 @@ private void updateSubscriptionsDispatcherRateLimiter() { protected CompletableFuture initTopicPolicy() { final var topicPoliciesService = brokerService.pulsar().getTopicPoliciesService(); final var partitionedTopicName = TopicName.getPartitionedTopicName(topic); - if (topicPoliciesService.registerListener(partitionedTopicName, this)) { + return topicPoliciesService.registerListenerAsync(partitionedTopicName, this).thenCompose(registered -> { + if (!registered) { + return CompletableFuture.completedFuture(null); + } if (ExtensibleLoadManagerImpl.isInternalTopic(topic)) { return CompletableFuture.completedFuture(null); } @@ -4791,8 +4794,7 @@ protected CompletableFuture initTopicPolicy() { TopicPoliciesService.GetType.LOCAL_ONLY)) .thenAcceptAsync(optionalPolicies -> optionalPolicies.ifPresent(this::onUpdate), brokerService.getTopicOrderedExecutor()); - } - return CompletableFuture.completedFuture(null); + }); } @VisibleForTesting diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/MetadataStoreTopicPoliciesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/MetadataStoreTopicPoliciesTest.java new file mode 100644 index 0000000000000..e7fefa164973a --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/MetadataStoreTopicPoliciesTest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.broker.admin; + +import org.apache.pulsar.broker.service.MetadataStoreTopicPoliciesService; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Test(groups = "broker-admin") +public class MetadataStoreTopicPoliciesTest extends TopicPoliciesTest { + + @BeforeClass(alwaysRun = true) + @Override + protected void setup() throws Exception { + conf.setTopicPoliciesServiceClassName(MetadataStoreTopicPoliciesService.class.getName()); + super.setup(); + } + + @Override + protected void clearTopicPoliciesCache() { + } + + @Test(enabled = false) + @Override + public void testTopicPolicyInitialValueWithNamespaceAlreadyLoaded() throws Exception { + // This test is specific to SystemTopicBasedTopicPoliciesService (uses getPoliciesCacheInit). + // Not applicable to MetadataStoreTopicPoliciesService. + } + + @Test(enabled = false) + @Override + public void testSystemTopicShouldBeCompacted() throws Exception { + // Relies on __change_events system topic, which does not exist with MetadataStoreTopicPoliciesService. + } + + @Test(enabled = false) + @Override + public void testPoliciesCanBeDeletedWithTopic() throws Exception { + // Directly accesses __change_events PersistentTopic for compaction. + // Not applicable to MetadataStoreTopicPoliciesService. + } + + @Test(enabled = false) + @Override + public void testProduceChangesWithEncryptionRequired() throws Exception { + // Checks __change_events LAC, which does not exist with MetadataStoreTopicPoliciesService. + } + + @Test(enabled = false) + @Override + public void testTopicPoliciesAfterCompaction(String reloadPolicyType) throws Exception { + // The "Recreate_Service" variant creates a new SystemTopicBasedTopicPoliciesService, + // which is not applicable to MetadataStoreTopicPoliciesService. + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java index c5537897e7bc7..0a1c76ff226ec 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java @@ -117,7 +117,9 @@ import org.glassfish.jersey.client.JerseyClientBuilder; import org.mockito.Mockito; import org.testng.Assert; +import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -142,10 +144,11 @@ public class TopicPoliciesTest extends MockedPulsarServiceBaseTest { private final int testTopicPartitions = 2; - @BeforeMethod + @BeforeClass(alwaysRun = true) @Override protected void setup() throws Exception { this.conf.setDefaultNumberOfNamespaceBundles(1); + this.conf.setForceDeleteNamespaceAllowed(true); super.internalSetup(); admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); @@ -156,15 +159,48 @@ protected void setup() throws Exception { admin.topics().createPartitionedTopic(testTopic, testTopicPartitions); Producer producer = pulsarClient.newProducer().topic(testTopic).create(); producer.close(); - waitForZooKeeperWatchers(); } - @AfterMethod(alwaysRun = true) + @AfterClass(alwaysRun = true) @Override public void cleanup() throws Exception { super.internalCleanup(); } + @BeforeMethod + void setupTestTopic() throws Exception { + // Recreate namespace to clear any policies set by previous tests + try { + admin.topics().deletePartitionedTopic(testTopic, true); + } catch (PulsarAdminException.NotFoundException e) { + // topic may already be deleted + } + try { + admin.namespaces().deleteNamespace(myNamespace, true); + } catch (PulsarAdminException.NotFoundException e) { + // namespace may already be deleted + } + try { + admin.namespaces().deleteNamespace(myNamespaceV1, true); + } catch (PulsarAdminException.NotFoundException e) { + // namespace may already be deleted + } + admin.namespaces().createNamespace(testTenant + "/" + testNamespace, Set.of("test")); + admin.namespaces().createNamespace(myNamespaceV1); + admin.topics().createPartitionedTopic(testTopic, testTopicPartitions); + // Acquire namespace bundle ownership so tests that call getOrCreateTopic() directly succeed. + // Without this, services that don't create a __change_events reader (e.g. MetadataStoreTopicPoliciesService) + // leave the bundle unowned after namespace recreation and the first broker-side topic load fails. + admin.lookups().lookupTopic(testTopic + "-partition-0"); + } + + @AfterMethod(alwaysRun = true) + void afterMethodCleanup() throws Exception{ + admin.brokers().updateDynamicConfiguration("maxPublishRatePerTopicInMessages", "0"); + admin.brokers().updateDynamicConfiguration("maxPublishRatePerTopicInBytes", "0"); + clearTopicPoliciesCache(); + } + @Test public void updatePropertiesForAutoCreatedTopicTest() throws Exception { TopicName topicName = TopicName.get( @@ -483,8 +519,8 @@ public Object[][] clientRequestType() { @Test(dataProvider = "clientRequestType") public void testPriorityOfGlobalPolicies(String clientRequestType) throws Exception { - final SystemTopicBasedTopicPoliciesService topicPoliciesService = - (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); + final TopicPoliciesService topicPoliciesService = + pulsar.getTopicPoliciesService(); final JerseyClient httpClient = JerseyClientBuilder.createClient(); // create topic and load it up. final String namespace = myNamespace; @@ -564,8 +600,8 @@ public void testPriorityOfGlobalPolicies(String clientRequestType) throws Except @Test(dataProvider = "clientRequestType") public void testPriorityOfGlobalPolicies2(String clientRequestType) throws Exception { - final SystemTopicBasedTopicPoliciesService topicPoliciesService = - (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); + final TopicPoliciesService topicPoliciesService = + pulsar.getTopicPoliciesService(); final JerseyClient httpClient = JerseyClientBuilder.createClient(); // create topic and load it up. final String namespace = myNamespace; @@ -651,8 +687,8 @@ public void testGlobalPolicyStillAffectsAfterUnloading() throws Exception { final TopicName topicName = TopicName.get(topic); admin.topics().createNonPartitionedTopic(topic); pulsarClient.newProducer().topic(topic).create().close(); - final SystemTopicBasedTopicPoliciesService topicPoliciesService = - (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); + final TopicPoliciesService topicPoliciesService = + pulsar.getTopicPoliciesService(); // Set non-global policy of the limitation of max consumers. // Set global policy of the limitation of max producers. @@ -693,8 +729,8 @@ public void testRetentionGlobalPolicyAffects() throws Exception { final TopicName topicName = TopicName.get(topic); admin.topics().createNonPartitionedTopic(topic); pulsarClient.newProducer().topic(topic).create().close(); - final SystemTopicBasedTopicPoliciesService topicPoliciesService = - (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); + final TopicPoliciesService topicPoliciesService = + pulsar.getTopicPoliciesService(); // Set non-global policy of the limitation of max consumers. // Set global policy of the persistence policies. @@ -2482,10 +2518,8 @@ public void testRemoveSubscribeRate() throws Exception { @Test public void testPublishRateInDifferentLevelPolicy() throws Exception { - cleanup(); - conf.setMaxPublishRatePerTopicInMessages(5); - conf.setMaxPublishRatePerTopicInBytes(50L); - setup(); + admin.brokers().updateDynamicConfiguration("maxPublishRatePerTopicInMessages", "5"); + admin.brokers().updateDynamicConfiguration("maxPublishRatePerTopicInBytes", "50"); final String topicName = "persistent://" + myNamespace + "/test-" + UUID.randomUUID(); pulsarClient.newProducer().topic(topicName).create().close(); @@ -2776,9 +2810,7 @@ public void testMaxSubscriptionsPerTopicWithExistingSubs() throws Exception { @Test public void testMaxUnackedMessagesOnSubscriptionPriority() throws Exception { - cleanup(); - conf.setMaxUnackedMessagesPerSubscription(30); - setup(); + restartBroker(conf -> conf.setMaxUnackedMessagesPerSubscription(30)); final String topic = "persistent://" + myNamespace + "/test-" + UUID.randomUUID(); // init cache @Cleanup @@ -2841,6 +2873,9 @@ public void testMaxUnackedMessagesOnSubscriptionPriority() throws Exception { && admin.topicPolicies().getMaxUnackedMessagesOnSubscription(topic) == null); messages = getMsgReceived(consumer1, Integer.MAX_VALUE); assertEquals(messages.size(), defaultMaxUnackedMsgOnBroker); + + // restore default config + restartBroker(conf -> conf.setMaxUnackedMessagesPerSubscription(4 * 50000)); } private void produceMsg(Producer producer, int msgNum) throws Exception{ @@ -3025,14 +3060,16 @@ public void testGetReplicatorRateApplied() throws Exception { @Test(timeOut = 30000) public void testAutoCreationDisabled() throws Exception { - cleanup(); - conf.setAllowAutoTopicCreation(false); - setup(); + admin.brokers().updateDynamicConfiguration("allowAutoTopicCreation", "false"); + final String topic = testTopic + UUID.randomUUID(); admin.topics().createPartitionedTopic(topic, 3); pulsarClient.newProducer().topic(topic).create().close(); //should not fail assertNull(admin.topicPolicies().getMessageTTL(topic)); + + // restore default + admin.brokers().updateDynamicConfiguration("allowAutoTopicCreation", "true"); } @Test @@ -3156,6 +3193,12 @@ public void testSubscriptionTypesEnabled() throws Exception { pulsarClient.newConsumer().topic(topic) .subscriptionType(SubscriptionType.Shared).subscriptionName("test") .subscribe().close(); + + // restore dynamic broker config and conf object + pulsar.getConfiguration().setSubscriptionTypesEnabled( + Set.of("Exclusive", "Shared", "Failover", "Key_Shared")); + admin.brokers().updateDynamicConfiguration("subscriptionTypesEnabled", + "Exclusive,Shared,Failover,Key_Shared"); } @Test(timeOut = 20000) @@ -3486,7 +3529,8 @@ public void testPolicyIsDeleteTogetherAutomatically() throws Exception { } @Test - public void testDoNotCreateSystemTopicForHeartbeatNamespace() { + public void testDoNotCreateSystemTopicForHeartbeatNamespace() throws Exception { + initEventsTopicAndPartitions(); assertTrue(pulsar.getBrokerService().getTopics().size() > 0); pulsar.getBrokerService().getTopics().forEach((k, v) -> { TopicName topicName = TopicName.get(k); @@ -3548,8 +3592,13 @@ public void testLoopCreateAndDeleteTopicPolicies() throws Exception { } private void triggerAndWaitNewTopicCompaction(String topicName) throws Exception { - PersistentTopic tp = - (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false).join().get(); + Optional topicOpt = + pulsar.getBrokerService().getTopic(topicName, false).join(); + if (topicOpt.isEmpty()) { + // Topic doesn't exist (e.g., when not using system-topic-based policies service), nothing to compact. + return; + } + PersistentTopic tp = (PersistentTopic) topicOpt.get(); // Wait for the old task finish. Awaitility.await().untilAsserted(() -> { CompletableFuture compactionTask = WhiteboxImpl.getInternalState(tp, "currentCompaction"); @@ -3568,7 +3617,7 @@ private void triggerAndWaitNewTopicCompaction(String topicName) throws Exception * It is not a thread safety method, something will go to a wrong pointer if there is a task is trying to load a * topic policies. */ - private void clearTopicPoliciesCache() { + protected void clearTopicPoliciesCache() { TopicPoliciesService topicPoliciesService = pulsar.getTopicPoliciesService(); if (topicPoliciesService instanceof TopicPoliciesService.TopicPoliciesServiceDisabled) { return; @@ -3798,8 +3847,8 @@ public void testGlobalTopicPolicies() throws Exception { .isNull()); admin.topicPolicies(true).setRetention(topic, new RetentionPolicies(1, 2)); - SystemTopicBasedTopicPoliciesService topicPoliciesService = - (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); + TopicPoliciesService topicPoliciesService = + pulsar.getTopicPoliciesService(); // check global topic policies can be added correctly. Awaitility.await().untilAsserted(() -> assertNotNull( @@ -3843,6 +3892,7 @@ public void testGlobalTopicPolicies() throws Exception { @Test public void testMaxMessageSizeWithChunking() throws Exception { + final var maxMessageSize = this.conf.getMaxMessageSize(); this.conf.setMaxMessageSize(1000); @Cleanup @@ -3871,6 +3921,7 @@ public void testMaxMessageSizeWithChunking() throws Exception { // chunk message send success producer.send(new byte[2000]); + this.conf.setMaxMessageSize(maxMessageSize); } @Test(timeOut = 30000) @@ -3924,6 +3975,7 @@ public void testGetTopicPoliciesWhenDeleteTopicPolicy() throws Exception { @Test public void testProduceChangesWithEncryptionRequired() throws Exception { + initEventsTopicAndPartitions(); final String beforeLac = admin.topics().getInternalStats(topicPolicyEventsTopic).lastConfirmedEntry; admin.namespaces().setEncryptionRequiredStatus(myNamespace, true); // just an update to trigger writes on __change_events @@ -4188,4 +4240,10 @@ public void testGetAppliedOffloadPoliciesWithLegacyNamespacePolicies() throws Ex assertEquals(offloadPolicies.getManagedLedgerOffloadThresholdInBytes(), (Long) (1024 * 1024 * 10L), "Should inherit offload threshold from legacy namespace policy"); } + + private void initEventsTopicAndPartitions() throws Exception { + try (Producer producer = pulsarClient.newProducer().topic(testTopic).create()) { + // No-op. Creating the producer initializes the events topic and partitions. + } + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/LegacyAwareTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/LegacyAwareTopicPoliciesServiceTest.java new file mode 100644 index 0000000000000..47a7de0528de9 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/LegacyAwareTopicPoliciesServiceTest.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.SystemTopicNames; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.TopicPolicies; +import org.awaitility.Awaitility; +import org.awaitility.core.ThrowingRunnable; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +/** + * Test order: testUpgrade() -> other tests (with MetadataStoreTopicPoliciesService configured) -> testDowngrade(). + */ +@Test(groups = "broker") +public class LegacyAwareTopicPoliciesServiceTest extends MockedPulsarServiceBaseTest { + + private static final String metaNamespace = "public/meta-ns"; + + @BeforeClass + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.setupDefaultTenantAndNamespace(); + } + + @AfterClass + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test(priority = -1) + public void testUpgrade() throws Exception { + final var topic = "test-upgrade"; + admin.topics().createNonPartitionedTopic(topic); + admin.topicPolicies().setCompactionThreshold(topic, 100); + waitUntilAssert(() -> assertEquals(admin.topicPolicies().getCompactionThreshold(topic), 100)); + + restartBroker(conf -> { + conf.setSystemTopicEnabled(false); + conf.setTopicPoliciesServiceClassName(MetadataStoreTopicPoliciesService.class.getName()); + }); + // The policies will be lost because when system topic is disabled, it will not try to read policies from the + // __change_events topic + assertNull(admin.topicPolicies().getCompactionThreshold(topic)); + + restartBroker(conf -> conf.setSystemTopicEnabled(true)); + // The default namespace still read policies from the __change_events topic if it exists + assertEquals(admin.topicPolicies().getCompactionThreshold(topic), 100); + assertFalse(pulsar.getLocalMetadataStore().exists(MetadataStoreTopicPoliciesService.LOCAL_POLICIES_ROOT).get()); + + // The global policies are still stored in the __change_events topic + admin.topicPolicies(true).setCompactionThreshold(topic, 200); + waitUntilAssert(() -> assertEquals(admin.topicPolicies(true).getCompactionThreshold(topic), 200)); + assertFalse(pulsar.getConfigurationMetadataStore() + .exists(MetadataStoreTopicPoliciesService.GLOBAL_POLICIES_ROOT).get()); + + admin.topicPolicies().deleteTopicPolicies(topic); + waitUntilAssert(() -> assertNull(admin.topicPolicies().getCompactionThreshold(topic))); + + admin.namespaces().createNamespace(metaNamespace); + } + + @Test(priority = 1) + public void testDowngrade() throws Exception { + final var topic1 = "downgrade"; // in default namespace + admin.topics().createNonPartitionedTopic(topic1); + admin.topicPolicies().setCompactionThreshold(topic1, 1); + waitUntilAssert(() -> assertEquals(admin.topicPolicies().getCompactionThreshold(topic1), 1)); + + final var topic2 = metaNamespace + "/downgrade"; + admin.topics().createNonPartitionedTopic(topic2); + admin.topicPolicies().setCompactionThreshold(topic2, 2); + waitUntilAssert(() -> assertEquals(admin.topicPolicies().getCompactionThreshold(topic2), 2)); + + restartBroker(conf -> + conf.setTopicPoliciesServiceClassName(SystemTopicBasedTopicPoliciesService.class.getName())); + assertEquals(admin.topicPolicies().getCompactionThreshold(topic1), 1); + // The policies will be lost because they are not stored in the __change_events topic + assertNull(admin.topicPolicies().getCompactionThreshold(topic2)); + } + + @DataProvider + public Object[][] namespaces() { + return new Object[][] { + { "public/default" }, + { metaNamespace } + }; + } + + @Test(dataProvider = "namespaces") + public void testPoliciesOperations(String namespace) throws Exception { + final var topicName = TopicName.get(namespace + "/test-policies-operations"); + final var topic = topicName.toString(); + admin.topics().createNonPartitionedTopic(topic); + + final var compactionThreshold = new AtomicLong(0); + // Verify the exception thrown from one listener does not affect other listeners + pulsar.getTopicPoliciesService().registerListenerAsync(topicName, __ -> { + throw new RuntimeException("injected failure"); + }).get(); + pulsar.getTopicPoliciesService().registerListenerAsync(topicName, policies -> + Optional.ofNullable(policies).map(TopicPolicies::getCompactionThreshold).ifPresentOrElse( + compactionThreshold::set, () -> compactionThreshold.set(-1))).get(); + + // Verify Created events are handled + admin.topicPolicies(false).setCompactionThreshold(topic, 100); + waitUntilAssert(() -> assertEquals(compactionThreshold.get(), 100)); + final var localStore = pulsar.getLocalMetadataStore(); + final var configurationStore = pulsar.getConfigurationMetadataStore(); + + if (namespace.equals(metaNamespace)) { + assertTrue(localStore.exists(MetadataStoreTopicPoliciesService.pathFor(topicName, false)).get()); + assertFalse(configurationStore.exists(MetadataStoreTopicPoliciesService.pathFor(topicName, true)).get()); + } + + admin.topicPolicies(true).setCompactionThreshold(topic, 200); + waitUntilAssert(() -> assertEquals(compactionThreshold.get(), 200)); + if (namespace.equals(metaNamespace)) { + assertTrue(configurationStore.exists(MetadataStoreTopicPoliciesService.pathFor(topicName, true)).get()); + } + + // Verify Modified events are handled + admin.topicPolicies(false).setCompactionThreshold(topic, 300); + waitUntilAssert(() -> assertEquals(compactionThreshold.get(), 300)); + + admin.topicPolicies(true).setCompactionThreshold(topic, 400); + waitUntilAssert(() -> assertEquals(compactionThreshold.get(), 400)); + + final var readerNamespaces = ((LegacyAwareTopicPoliciesService) pulsar.getTopicPoliciesService()) + .systemTopicService.getReaderCaches().keySet(); + assertFalse(readerNamespaces.contains(NamespaceName.get(metaNamespace))); + + // Verify Deleted events are handled + admin.topicPolicies(false).deleteTopicPolicies(topic); + waitUntilAssert(() -> assertEquals(compactionThreshold.get(), -1)); + if (namespace.equals(metaNamespace)) { + assertFalse(localStore.exists(MetadataStoreTopicPoliciesService.pathFor(topicName, false)).get()); + assertFalse(configurationStore.exists(MetadataStoreTopicPoliciesService.pathFor(topicName, true)).get()); + } + } + + @Test + public void testUserCreatedEventsTopicAreIgnored() throws Exception { + final var topic = TopicName.get(metaNamespace + "/" + System.currentTimeMillis()).toString(); + admin.topics().createNonPartitionedTopic(topic); + admin.topicPolicies().setCompactionThreshold(topic, 1); + waitUntilAssert(() -> assertEquals(admin.topicPolicies().getCompactionThreshold(topic), 1)); + + final var eventsTopic = metaNamespace + "/" + SystemTopicNames.NAMESPACE_EVENTS_LOCAL_NAME; + admin.topics().createNonPartitionedTopic(eventsTopic); + // Even if the __change_events topic is created, since it has detected the namespace didn't have the events + // topic before, it will be ignored and the policies are still read from metadata store. + waitUntilAssert(() -> assertEquals(admin.topicPolicies().getCompactionThreshold(topic), 1)); + admin.topics().delete(eventsTopic); + } + + private static void waitUntilAssert(ThrowingRunnable assertion) { + Awaitility.await().atMost(Duration.ofSeconds(1)).untilAsserted(assertion); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java index ad02b4707a3bb..8d3b16723ed4f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java @@ -110,7 +110,7 @@ public void onUpdate(TopicPolicies data) { CompletableFuture f = CompletableFuture.completedFuture(null).thenRunAsync(() -> { for (int i = 0; i < 100; i++) { TopicPolicyListener listener = new TopicPolicyListenerImpl(); - systemTopicBasedTopicPoliciesService.registerListener(topicName, listener); + systemTopicBasedTopicPoliciesService.registerListenerAsync(topicName, listener); Assert.assertNotNull(systemTopicBasedTopicPoliciesService.listeners.get(topicName)); Assert.assertTrue(systemTopicBasedTopicPoliciesService.listeners.get(topicName).size() >= 1); systemTopicBasedTopicPoliciesService.unregisterListener(topicName, listener); @@ -119,7 +119,7 @@ public void onUpdate(TopicPolicies data) { for (int i = 0; i < 100; i++) { TopicPolicyListener listener = new TopicPolicyListenerImpl(); - systemTopicBasedTopicPoliciesService.registerListener(topicName, listener); + systemTopicBasedTopicPoliciesService.registerListenerAsync(topicName, listener); Assert.assertNotNull(systemTopicBasedTopicPoliciesService.listeners.get(topicName)); Assert.assertTrue(systemTopicBasedTopicPoliciesService.listeners.get(topicName).size() >= 1); systemTopicBasedTopicPoliciesService.unregisterListener(topicName, listener); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPolicyTestUtils.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPolicyTestUtils.java index 6b9735d59b21a..7e9c697fb5d03 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPolicyTestUtils.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPolicyTestUtils.java @@ -72,6 +72,13 @@ public static TopicPolicies getGlobalTopicPolicies(TopicPoliciesService topicPol public static Optional getTopicPoliciesBypassCache(TopicPoliciesService topicPoliciesService, TopicName topicName, boolean isGlobal) throws Exception { + if (topicPoliciesService instanceof LegacyAwareTopicPoliciesService legacyService) { + TopicPoliciesService resolved = legacyService.resolveService(topicName.getNamespaceObject()).get(); + return getTopicPoliciesBypassCache(resolved, topicName, isGlobal); + } + if (topicPoliciesService instanceof MetadataStoreTopicPoliciesService metadataStoreService) { + return metadataStoreService.getTopicPoliciesDirectFromStore(topicName, isGlobal).get(); + } @Cleanup final var reader = ((SystemTopicBasedTopicPoliciesService) topicPoliciesService) .getNamespaceEventsSystemTopicFactory() .createTopicPoliciesSystemTopicClient(topicName.getNamespaceObject()) From 1885cf17a1bff63ab77b9ab94ab800b15a5e8cc6 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 8 Jun 2026 15:18:31 +0800 Subject: [PATCH 044/213] Fix build failure due to an old version of Caffine --- .../broker/service/LegacyAwareTopicPoliciesService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/LegacyAwareTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/LegacyAwareTopicPoliciesService.java index 6e1bfd7bacc42..c9d6fbd3ac3ef 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/LegacyAwareTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/LegacyAwareTopicPoliciesService.java @@ -56,8 +56,8 @@ public LegacyAwareTopicPoliciesService(PulsarService pulsar, .buildAsync(new AsyncCacheLoader<>() { @NonNull @Override - public CompletableFuture asyncLoad(NamespaceName key, - @NonNull Executor executor) { + public CompletableFuture asyncLoad(NamespaceName key, + @NonNull Executor executor) { return NamespaceEventsSystemTopicFactory.checkSystemTopicExists(key, EventType.TOPIC_POLICY, pulsar); } From 500d85cf065c49846f26a23a666c9609a339a689 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Sun, 7 Jun 2026 23:10:15 +0300 Subject: [PATCH 045/213] [fix][broker] Fix tableview divergence in ServiceUnitStateTableViewSyncer causing flaky tests (#25946) --- .../ServiceUnitStateTableViewSyncer.java | 188 ++++++++-- .../ExtensibleLoadManagerImplBaseTest.java | 16 +- .../ExtensibleLoadManagerImplTest.java | 320 ++++++++++++------ 3 files changed, 376 insertions(+), 148 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewSyncer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewSyncer.java index 45ff0dcb2674e..999c7bea93ad2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewSyncer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewSyncer.java @@ -23,20 +23,26 @@ import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.COMPACTION_THRESHOLD; import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.configureSystemTopics; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.google.common.annotations.VisibleForTesting; import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.function.BiConsumer; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.jspecify.annotations.NonNull; /** * Defines ServiceUnitTableViewSyncer. @@ -47,10 +53,15 @@ public class ServiceUnitStateTableViewSyncer implements Closeable { private static final int MAX_CONCURRENT_SYNC_COUNT = 100; private static final int SYNC_WAIT_TIME_IN_SECS = 300; + private static final long RECONCILE_INTERVAL_IN_MILLIS = 5_000; + private static final BiConsumer NOOP_CONSUMER = (__, ___) -> { + }; + private volatile int syncWaitTimeInSecs = SYNC_WAIT_TIME_IN_SECS; private PulsarService pulsar; private volatile ServiceUnitStateTableView systemTopicTableView; private volatile ServiceUnitStateTableView metadataStoreTableView; private volatile boolean isActive = false; + private final ObjectWriter jsonWriter = ObjectMapperFactory.getMapper().writer(); public void start(PulsarService pulsar) @@ -82,53 +93,77 @@ public void start(PulsarService pulsar) } private CompletableFuture syncToSystemTopic(String key, ServiceUnitStateData data) { - return systemTopicTableView.put(key, data); + return logIfFailed(sync(systemTopicTableView, key, data), key, data, "systemTopic"); } private CompletableFuture syncToMetadataStore(String key, ServiceUnitStateData data) { - return metadataStoreTableView.put(key, data); + return logIfFailed(sync(metadataStoreTableView, key, data), key, data, "metadataStore"); } - private void dummy(String key, ServiceUnitStateData data) { + private CompletableFuture sync(ServiceUnitStateTableView dst, String key, ServiceUnitStateData data) { + // A null tail item is a tombstone: the source view removed the key. Route it to + // delete() rather than put(): the metadata-store view's put() rejects null + // (@NonNull) and the system-topic view's delete() is itself a null-valued put(), + // so a uniform delete keeps both sync directions symmetric and prevents a missed + // deletion from leaving the two views with different sizes (which would make + // waitUntilSynced spin until the timeout budget). + return data == null ? dst.delete(key) : dst.put(key, data); + } + + private CompletableFuture logIfFailed(CompletableFuture future, String key, + ServiceUnitStateData data, String dst) { + return future.whenComplete((__, e) -> { + if (e != null && !(e instanceof PulsarClientException.AlreadyClosedException)) { + log.warn("Failed to sync tableview item; sizes may diverge until the next update;" + + " dst={} serviceUnit={} data={}", dst, key, data, e); + } + }); } private void syncExistingItems() throws IOException, ExecutionException, InterruptedException, TimeoutException { long started = System.currentTimeMillis(); + @Cleanup ServiceUnitStateTableView metadataStoreTableView = new ServiceUnitStateMetadataStoreTableViewImpl(); metadataStoreTableView.start( pulsar, - this::dummy, - this::dummy, - this::dummy + NOOP_CONSUMER, + NOOP_CONSUMER, + NOOP_CONSUMER ); @Cleanup ServiceUnitStateTableView systemTopicTableView = new ServiceUnitStateTableViewImpl(); systemTopicTableView.start( pulsar, - this::dummy, - this::dummy, - this::dummy + NOOP_CONSUMER, + NOOP_CONSUMER, + NOOP_CONSUMER ); var syncer = pulsar.getConfiguration().getLoadBalancerServiceUnitTableViewSyncer(); + ServiceUnitStateTableView src; + ServiceUnitStateTableView dst; if (syncer == SystemTopicToMetadataStoreSyncer) { clean(metadataStoreTableView); syncExistingItemsToMetadataStore(systemTopicTableView); + src = systemTopicTableView; + dst = metadataStoreTableView; } else { clean(systemTopicTableView); syncExistingItemsToSystemTopic(metadataStoreTableView, systemTopicTableView); + src = metadataStoreTableView; + dst = systemTopicTableView; } - if (!waitUntilSynced(metadataStoreTableView, systemTopicTableView, started)) { + if (!waitUntilSynced(src, dst, started)) { throw new TimeoutException( syncer + " failed to sync existing items in tableviews. MetadataStoreTableView.size: " + metadataStoreTableView.entrySet().size() + ", SystemTopicTableView.size: " + systemTopicTableView.entrySet().size() + " in " - + SYNC_WAIT_TIME_IN_SECS + " secs"); + + syncWaitTimeInSecs + " secs"); } log.info("Synced existing items MetadataStoreTableView.size:{} , " @@ -154,8 +189,8 @@ private void syncTailItems() throws InterruptedException, IOException, TimeoutEx this.metadataStoreTableView.start( pulsar, this::syncToSystemTopic, - this::dummy, - this::dummy + NOOP_CONSUMER, + NOOP_CONSUMER ); log.info("Started MetadataStoreTableView"); @@ -163,18 +198,20 @@ private void syncTailItems() throws InterruptedException, IOException, TimeoutEx this.systemTopicTableView.start( pulsar, this::syncToMetadataStore, - this::dummy, - this::dummy + NOOP_CONSUMER, + NOOP_CONSUMER ); log.info("Started SystemTopicTableView"); var syncer = pulsar.getConfiguration().getLoadBalancerServiceUnitTableViewSyncer(); - if (!waitUntilSynced(metadataStoreTableView, systemTopicTableView, started)) { + var src = syncer == SystemTopicToMetadataStoreSyncer ? systemTopicTableView : metadataStoreTableView; + var dst = syncer == SystemTopicToMetadataStoreSyncer ? metadataStoreTableView : systemTopicTableView; + if (!waitUntilSynced(src, dst, started)) { throw new TimeoutException( syncer + " failed to sync tableviews. MetadataStoreTableView.size: " + metadataStoreTableView.entrySet().size() + ", SystemTopicTableView.size: " + systemTopicTableView.entrySet().size() + " in " - + SYNC_WAIT_TIME_IN_SECS + " secs"); + + syncWaitTimeInSecs + " secs"); } @@ -187,62 +224,134 @@ private void syncTailItems() throws InterruptedException, IOException, TimeoutEx private void syncExistingItemsToMetadataStore(ServiceUnitStateTableView src) throws JsonProcessingException, ExecutionException, InterruptedException, TimeoutException { // Directly use store to sync existing items to metadataStoreTableView(otherwise, they are conflicted out) - var store = pulsar.getLocalMetadataStore(); - var writer = ObjectMapperFactory.getMapper().writer(); - var opTimeout = pulsar.getConfiguration().getMetadataStoreOperationTimeoutSeconds(); List> futures = new ArrayList<>(); var srcIter = src.entrySet().iterator(); while (srcIter.hasNext()) { var e = srcIter.next(); - futures.add(store.put(ServiceUnitStateMetadataStoreTableViewImpl.PATH_PREFIX + "/" + e.getKey(), - writer.writeValueAsBytes(e.getValue()), Optional.empty()).thenApply(__ -> null)); - if (futures.size() == MAX_CONCURRENT_SYNC_COUNT || !srcIter.hasNext()) { - FutureUtil.waitForAll(futures).get(opTimeout, TimeUnit.SECONDS); - } + futures.add(writeToMetadataStore(e.getKey(), e.getValue())); + maybeWaitCompletion(futures, !srcIter.hasNext()); + } + } + + private void maybeWaitCompletion(List> futures, boolean forceWait) + throws InterruptedException, ExecutionException, TimeoutException { + if (!futures.isEmpty() && (futures.size() == MAX_CONCURRENT_SYNC_COUNT || forceWait)) { + FutureUtil.waitForAll(futures) + .get(pulsar.getConfiguration().getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS); + futures.clear(); } } + private @NonNull CompletableFuture writeToMetadataStore(String key, ServiceUnitStateData value) + throws JsonProcessingException { + return pulsar.getLocalMetadataStore().put(ServiceUnitStateMetadataStoreTableViewImpl.PATH_PREFIX + "/" + key, + jsonWriter.writeValueAsBytes(value), Optional.empty()).thenApply(__ -> null); + } + private void syncExistingItemsToSystemTopic(ServiceUnitStateTableView src, ServiceUnitStateTableView dst) throws ExecutionException, InterruptedException, TimeoutException { - var opTimeout = pulsar.getConfiguration().getMetadataStoreOperationTimeoutSeconds(); List> futures = new ArrayList<>(); var srcIter = src.entrySet().iterator(); while (srcIter.hasNext()) { var e = srcIter.next(); futures.add(dst.put(e.getKey(), e.getValue())); - if (futures.size() == MAX_CONCURRENT_SYNC_COUNT || !srcIter.hasNext()) { - FutureUtil.waitForAll(futures).get(opTimeout, TimeUnit.SECONDS); - } + maybeWaitCompletion(futures, !srcIter.hasNext()); } } private void clean(ServiceUnitStateTableView dst) throws ExecutionException, InterruptedException, TimeoutException { - var opTimeout = pulsar.getConfiguration().getMetadataStoreOperationTimeoutSeconds(); var dstIter = dst.entrySet().iterator(); List> futures = new ArrayList<>(); while (dstIter.hasNext()) { var e = dstIter.next(); futures.add(dst.delete(e.getKey())); - if (futures.size() == MAX_CONCURRENT_SYNC_COUNT || !dstIter.hasNext()) { - FutureUtil.waitForAll(futures).get(opTimeout, TimeUnit.SECONDS); - } + maybeWaitCompletion(futures, !dstIter.hasNext()); } } - private boolean waitUntilSynced(ServiceUnitStateTableView srt, ServiceUnitStateTableView dst, long started) + private boolean waitUntilSynced(ServiceUnitStateTableView src, ServiceUnitStateTableView dst, long started) throws InterruptedException { - while (srt.entrySet().size() != dst.entrySet().size()) { - if (TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - started) - > SYNC_WAIT_TIME_IN_SECS) { + long lastReconciled = started; + while (src.entrySet().size() != dst.entrySet().size()) { + long now = System.currentTimeMillis(); + if (TimeUnit.MILLISECONDS.toSeconds(now - started) > syncWaitTimeInSecs) { return false; } + // Give in-flight syncs a grace period to settle on their own, then reconcile + // periodically: updates that raced with the table views' (re)start were replayed + // to the fresh views as existing items — which are deliberately not wired to + // sync — so without reconciliation the views would never converge. + if (now - lastReconciled >= RECONCILE_INTERVAL_IN_MILLIS) { + if (log.isDebugEnabled()) { + log.debug("Tableviews not synced yet; reconciling; srcSize={} dstSize={} elapsedSecs={}", + src.entrySet().size(), dst.entrySet().size(), + TimeUnit.MILLISECONDS.toSeconds(now - started)); + } + reconcile(src, dst, started); + lastReconciled = now; + } Thread.sleep(100); } return true; } + /** + * Copies items the destination table view is missing and removes stale items that no longer + * exist in the source. Channel updates that land between the existing-items copy and the + * registration of the tail listeners are only visible as existing items of the freshly + * started views, so the tail listeners never see them. Writes flow to the migration source + * while the syncer starts, making the source view authoritative; destination-only items are + * removed only when they predate this sync phase and are still absent from the source, so a + * concurrent fresh write to the destination is never discarded. Failures are logged and left + * for the next reconcile pass. Runs on the caller's (load manager) thread with each batch + * bounded by the metadata store operation timeout. + */ + private void reconcile(ServiceUnitStateTableView src, ServiceUnitStateTableView dst, long started) + throws InterruptedException { + // Snapshot the destination entries before iterating the source so that a key arriving + // in the destination through a concurrent tail sync cannot be misclassified as stale. + var staleDstItems = new HashMap(); + for (var e : dst.entrySet()) { + staleDstItems.put(e.getKey(), e.getValue()); + } + try { + List> futures = new ArrayList<>(); + for (var e : src.entrySet()) { + if (staleDstItems.remove(e.getKey()) == null) { + log.info("Reconciling item missing from the destination tableview; serviceUnit={}", + e.getKey()); + if (dst.isMetadataStoreBased()) { + // Write directly to the store like syncExistingItemsToMetadataStore + // does; the view's put() would conflict the item out. + futures.add(writeToMetadataStore(e.getKey(), e.getValue())); + } else { + futures.add(dst.put(e.getKey(), e.getValue())); + } + maybeWaitCompletion(futures, false); + } + } + for (var e : staleDstItems.entrySet()) { + // Only remove items written before this sync phase began and re-confirmed absent + // from the source: a fresh destination write (e.g. from a broker already switched + // to the destination implementation) is propagated to the source by the tail + // listener instead of being deleted here. + if (e.getValue().timestamp() < started && src.get(e.getKey()) == null) { + log.info("Reconciling stale item in the destination tableview; serviceUnit={}", + e.getKey()); + futures.add(dst.delete(e.getKey())); + maybeWaitCompletion(futures, false); + } + } + maybeWaitCompletion(futures, true); + } catch (IOException | ExecutionException | TimeoutException e) { + // Transient write failures leave a size divergence behind; the next reconcile pass + // (or the sync-wait timeout) handles it. + log.warn("Failed to reconcile tableview items", e); + } + } + @Override public void close() throws IOException { if (!isActive) { @@ -282,4 +391,9 @@ public void close() throws IOException { public boolean isActive() { return isActive; } + + @VisibleForTesting + public void setSyncWaitTimeInSecs(int syncWaitTimeInSecs) { + this.syncWaitTimeInSecs = syncWaitTimeInSecs; + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplBaseTest.java index 55e9b8d6baf54..2b30723f0a26a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplBaseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplBaseTest.java @@ -47,6 +47,7 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.awaitility.Awaitility; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; @@ -196,7 +197,20 @@ protected void cleanup() throws Exception { @BeforeMethod(alwaysRun = true) protected void initializeState() throws PulsarAdminException, IllegalAccessException { - admin.namespaces().unload(defaultTestNamespace); + // After a prior test churned leader election, the channel-topic bundle can be left + // unserved ("not served by this instance"), making the unload's channel publish fail + // (HTTP 500) or hang server-side until the background monitor task (120s interval) + // reconciles the brokers' roles with the channel ownership. Drive monitor() eagerly to + // heal that state, bound each unload attempt (a synchronous unload() can block longer + // than the whole retry window), and fail loudly on exhaustion. + Awaitility.await().atMost(120, TimeUnit.SECONDS) + .pollInterval(1, TimeUnit.SECONDS) + .ignoreExceptions() + .untilAsserted(() -> { + primaryLoadManager.monitor(); + secondaryLoadManager.monitor(); + admin.namespaces().unloadAsync(defaultTestNamespace).get(15, TimeUnit.SECONDS); + }); reset(primaryLoadManager, secondaryLoadManager); FieldUtils.writeDeclaredField(pulsarClient, "lookup", lookupService, true); pulsar1.getConfig().setLoadBalancerMultiPhaseBundleUnload(true); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index 65d017499fdb0..05e9bfba6efcf 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -1331,7 +1331,11 @@ public void testDeployAndRollbackLoadManager() throws Exception { } } - @Test(priority = 200) + // Cap below the 300s suite default (AnnotationListener) so a hung ServiceUnitStateTableViewSyncer + // start() fails fast and is retried instead of consuming the full 300s slot and corrupting the + // next @BeforeMethod. Combined with the shortened sync-wait budget set below, a real divergence + // surfaces within the shortened budget instead of a 5-minute ThreadTimeoutException. + @Test(priority = 200, timeOut = 240 * 1000) public void testLoadBalancerServiceUnitTableViewSyncer() throws Exception { // Make pulsar1 the leader so primaryLoadManager is the syncer-running broker. makePrimaryAsLeader(); @@ -1361,120 +1365,142 @@ public void testLoadBalancerServiceUnitTableViewSyncer() throws Exception { String syncerType = serviceUnitStateTableViewClassName.equals(ServiceUnitStateTableViewImpl.class.getName()) ? "SystemTopicToMetadataStoreSyncer" : "MetadataStoreToSystemTopicSyncer"; + // Shrink the sync-wait budget on both brokers' live syncers BEFORE the first start() + // (driven by monitor() below) so any tableview-size divergence fails in ~30s with the + // syncer's own TimeoutException instead of spinning for the full 300s default — the + // exact hang observed in CI happened inside that first start(). + primaryLoadManager.getServiceUnitStateTableViewSyncer().setSyncWaitTimeInSecs(30); + secondaryLoadManager.getServiceUnitStateTableViewSyncer().setSyncWaitTimeInSecs(30); + pulsar.getAdminClient().brokers() .updateDynamicConfiguration("loadBalancerServiceUnitTableViewSyncer", syncerType); Awaitility.await().untilAsserted(() -> assertTrue(pulsar1.getConfiguration().isLoadBalancerServiceUnitTableViewSyncerEnabled())); - primaryLoadManager.monitor(); - Awaitility.await().atMost(30, TimeUnit.SECONDS) - .untilAsserted(() -> assertTrue(primaryLoadManager.getServiceUnitStateTableViewSyncer() - .isActive())); - assertFalse(secondaryLoadManager.getServiceUnitStateTableViewSyncer().isActive()); - - // === Phase 2: add a 3rd broker using the OTHER table view impl === - // pulsar1/pulsar2 use serviceUnitStateTableViewClassName; pulsar3 deliberately - // uses the other one so the test exercises cross-impl lookups regardless of - // which parametrization we're running. - String otherClassName = - serviceUnitStateTableViewClassName.equals(ServiceUnitStateTableViewImpl.class.getName()) - ? ServiceUnitStateMetadataStoreTableViewImpl.class.getName() - : ServiceUnitStateTableViewImpl.class.getName(); - - ServiceConfiguration crossImplConf = getDefaultConf(); - crossImplConf.setAllowAutoTopicCreation(true); - crossImplConf.setForceDeleteNamespaceAllowed(true); - crossImplConf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getCanonicalName()); - crossImplConf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); - crossImplConf.setLoadManagerServiceUnitStateTableViewClassName(otherClassName); - - try (var crossImplCtx = createAdditionalPulsarTestContext(crossImplConf)) { - var pulsar3 = crossImplCtx.getPulsarService(); - - // All three brokers (across both impls) must agree on topic ownership. - assertEquals(pulsar2.getAdminClient().lookups().lookupTopic(topic), ownershipBefore); - assertEquals(pulsar3.getAdminClient().lookups().lookupTopic(topic), ownershipBefore); - Optional webUrlPulsar3 = pulsar3.getNamespaceService().getWebServiceUrl(bundle, options); - assertTrue(webUrlPulsar3.isPresent()); - assertEquals(webUrlPulsar3.get().toString(), webUrlBefore.get().toString()); - - // SLA monitor and heartbeat lookups must agree across impls in every direction. - List brokers = List.of(pulsar1, pulsar2, pulsar3); - for (PulsarService viewer : brokers) { - for (PulsarService owner : brokers) { - assertLookupHeartbeatOwner(viewer, owner.getBrokerId(), owner.getBrokerServiceUrl()); - assertLookupSLANamespaceOwner(viewer, owner.getBrokerId(), owner.getBrokerServiceUrl()); + // Drive monitor() inside the await so a transient start() failure (swallowed by + // monitor()'s catch) is retried instead of waiting for the 120s background monitor task. + Awaitility.await().atMost(120, TimeUnit.SECONDS).untilAsserted(() -> { + primaryLoadManager.monitor(); + assertTrue(primaryLoadManager.getServiceUnitStateTableViewSyncer().isActive()); + }); + + try { + assertFalse(secondaryLoadManager.getServiceUnitStateTableViewSyncer().isActive()); + + // === Phase 2: add a 3rd broker using the OTHER table view impl === + // pulsar1/pulsar2 use serviceUnitStateTableViewClassName; pulsar3 deliberately + // uses the other one so the test exercises cross-impl lookups regardless of + // which parametrization we're running. + String otherClassName = + serviceUnitStateTableViewClassName.equals(ServiceUnitStateTableViewImpl.class.getName()) + ? ServiceUnitStateMetadataStoreTableViewImpl.class.getName() + : ServiceUnitStateTableViewImpl.class.getName(); + + ServiceConfiguration crossImplConf = getDefaultConf(); + crossImplConf.setAllowAutoTopicCreation(true); + crossImplConf.setForceDeleteNamespaceAllowed(true); + crossImplConf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getCanonicalName()); + crossImplConf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); + crossImplConf.setLoadManagerServiceUnitStateTableViewClassName(otherClassName); + + try (var crossImplCtx = createAdditionalPulsarTestContext(crossImplConf)) { + var pulsar3 = crossImplCtx.getPulsarService(); + + // All three brokers (across both impls) must agree on topic ownership. + assertEquals(pulsar2.getAdminClient().lookups().lookupTopic(topic), ownershipBefore); + assertEquals(pulsar3.getAdminClient().lookups().lookupTopic(topic), ownershipBefore); + Optional webUrlPulsar3 = pulsar3.getNamespaceService().getWebServiceUrl(bundle, options); + assertTrue(webUrlPulsar3.isPresent()); + assertEquals(webUrlPulsar3.get().toString(), webUrlBefore.get().toString()); + + // SLA monitor and heartbeat lookups must agree across impls in every direction. + List brokers = List.of(pulsar1, pulsar2, pulsar3); + for (PulsarService viewer : brokers) { + for (PulsarService owner : brokers) { + assertLookupHeartbeatOwner(viewer, owner.getBrokerId(), owner.getBrokerServiceUrl()); + assertLookupSLANamespaceOwner(viewer, owner.getBrokerId(), owner.getBrokerServiceUrl()); + } } - } - // === Phase 3: simulate the cross-impl broker going offline === - // Its SLA namespace must reassign to a remaining broker, and the ownership - // change must propagate through the syncer to brokers using the other impl. - var wrapper3 = (ExtensibleLoadManagerWrapper) pulsar3.getLoadManager().get(); - var loadManager3 = (ExtensibleLoadManagerImpl) - FieldUtils.readField(wrapper3, "loadManager", true); - ServiceUnitStateChannel channel3 = (ServiceUnitStateChannel) - FieldUtils.readField(loadManager3, "serviceUnitStateChannel", true); - channel3.cleanOwnerships(); - // Set state to Closed BEFORE deleting the ZK node to prevent the notification - // handler's session-expiry recovery from auto-re-registering broker3. In - // production the PulsarService shuts down after unregister(), so the handler - // never fires; in tests the service stays running and creates a race. - var registry3 = (BrokerRegistryImpl) loadManager3.getBrokerRegistry(); - registry3.state.set(BrokerRegistryImpl.State.Closed); - pulsar3.getLocalMetadataStore() - .delete("/loadbalance/brokers/" + pulsar3.getBrokerId(), Optional.empty()).get(); - - String slaMonitorTopic = getSLAMonitorNamespace(pulsar3.getBrokerId(), pulsar.getConfiguration()) - .getPersistentTopicName("test"); - String pulsar3BrokerUrl = pulsar3.getBrokerServiceUrl(); - Awaitility.await().atMost(30, TimeUnit.SECONDS).ignoreExceptions().untilAsserted(() -> { - String reassigned = pulsar.getAdminClient().lookups().lookupTopic(slaMonitorTopic); - assertNotNull(reassigned); - assertNotEquals(reassigned, pulsar3BrokerUrl); - }); + // === Phase 3: simulate the cross-impl broker going offline === + // Its SLA namespace must reassign to a remaining broker, and the ownership + // change must propagate through the syncer to brokers using the other impl. + var wrapper3 = (ExtensibleLoadManagerWrapper) pulsar3.getLoadManager().get(); + var loadManager3 = (ExtensibleLoadManagerImpl) + FieldUtils.readField(wrapper3, "loadManager", true); + ServiceUnitStateChannel channel3 = (ServiceUnitStateChannel) + FieldUtils.readField(loadManager3, "serviceUnitStateChannel", true); + channel3.cleanOwnerships(); + // Set state to Closed BEFORE deleting the ZK node to prevent the notification + // handler's session-expiry recovery from auto-re-registering broker3. In + // production the PulsarService shuts down after unregister(), so the handler + // never fires; in tests the service stays running and creates a race. + var registry3 = (BrokerRegistryImpl) loadManager3.getBrokerRegistry(); + registry3.state.set(BrokerRegistryImpl.State.Closed); + pulsar3.getLocalMetadataStore() + .delete("/loadbalance/brokers/" + pulsar3.getBrokerId(), Optional.empty()).get(); + + String slaMonitorTopic = getSLAMonitorNamespace(pulsar3.getBrokerId(), pulsar.getConfiguration()) + .getPersistentTopicName("test"); + String pulsar3BrokerUrl = pulsar3.getBrokerServiceUrl(); + Awaitility.await().atMost(30, TimeUnit.SECONDS).ignoreExceptions().untilAsserted(() -> { + String reassigned = pulsar.getAdminClient().lookups().lookupTopic(slaMonitorTopic); + assertNotNull(reassigned); + assertNotEquals(reassigned, pulsar3BrokerUrl); + }); - // Send a message while the topic is owned by the reassigned broker; this must - // remain durable when ownership migrates back below. - @Cleanup - Producer producer = pulsar.getClient().newProducer(Schema.STRING) - .topic(slaMonitorTopic).create(); - producer.send("offline"); - - // === Phase 4: re-register the broker and verify ownership returns === - registry3.state.set(BrokerRegistryImpl.State.Started); - registry3.registerAsync().get(); - Awaitility.await().atMost(30, TimeUnit.SECONDS).ignoreExceptions().untilAsserted(() -> - assertEquals(pulsar.getAdminClient().lookups().lookupTopic(slaMonitorTopic), - pulsar3.getBrokerServiceUrl())); - - // Same producer reconnects to the new owner; a fresh producer also works. - producer.send("after-reconnect"); - @Cleanup - Producer producer2 = pulsar.getClient().newProducer(Schema.STRING) - .topic(slaMonitorTopic).create(); - producer2.send("from-new-producer"); + // Send a message while the topic is owned by the reassigned broker; this must + // remain durable when ownership migrates back below. + @Cleanup + Producer producer = pulsar.getClient().newProducer(Schema.STRING) + .topic(slaMonitorTopic).create(); + producer.send("offline"); - @Cleanup - Consumer consumer = pulsar.getClient().newConsumer(Schema.STRING) - .topic(slaMonitorTopic) - .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) - .subscriptionName("test") - .subscribe(); - assertEquals(consumer.receive().getValue(), "offline"); - assertEquals(consumer.receive().getValue(), "after-reconnect"); - assertEquals(consumer.receive().getValue(), "from-new-producer"); - } + // === Phase 4: re-register the broker and verify ownership returns === + registry3.state.set(BrokerRegistryImpl.State.Started); + registry3.registerAsync().get(); + Awaitility.await().atMost(30, TimeUnit.SECONDS).ignoreExceptions().untilAsserted(() -> + assertEquals(pulsar.getAdminClient().lookups().lookupTopic(slaMonitorTopic), + pulsar3.getBrokerServiceUrl())); - // === Phase 5: disable the syncer and verify it deactivates === - pulsar.getAdminClient().brokers() - .deleteDynamicConfiguration("loadBalancerServiceUnitTableViewSyncer"); - Awaitility.await().untilAsserted(() -> - assertFalse(pulsar1.getConfiguration().isLoadBalancerServiceUnitTableViewSyncerEnabled())); - primaryLoadManager.monitor(); - Awaitility.await().atMost(30, TimeUnit.SECONDS) - .untilAsserted(() -> assertFalse(primaryLoadManager.getServiceUnitStateTableViewSyncer() - .isActive())); - assertFalse(secondaryLoadManager.getServiceUnitStateTableViewSyncer().isActive()); + // Same producer reconnects to the new owner; a fresh producer also works. + producer.send("after-reconnect"); + @Cleanup + Producer producer2 = pulsar.getClient().newProducer(Schema.STRING) + .topic(slaMonitorTopic).create(); + producer2.send("from-new-producer"); + + @Cleanup + Consumer consumer = pulsar.getClient().newConsumer(Schema.STRING) + .topic(slaMonitorTopic) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscriptionName("test") + .subscribe(); + assertEquals(consumer.receive().getValue(), "offline"); + assertEquals(consumer.receive().getValue(), "after-reconnect"); + assertEquals(consumer.receive().getValue(), "from-new-producer"); + } + } finally { + // === Phase 5: disable the syncer and verify it deactivates === + // Guarantee the dynamic config is removed and the syncer is driven inactive even if + // the body threw, so the syncer cannot stay enabled and poison later tests. Note this + // cannot tear down a start() that failed before isActive=true (close() short-circuits + // on !isActive); leftover tail views from such a partial start are recovered by the + // next successful start(), and the next test's initializeState() retry absorbs any + // residual channel disruption. + try { + pulsar.getAdminClient().brokers() + .deleteDynamicConfiguration("loadBalancerServiceUnitTableViewSyncer"); + } catch (Exception e) { + log.warn("Failed to delete syncer dynamic config in cleanup", e); + } + Awaitility.await().atMost(60, TimeUnit.SECONDS).ignoreExceptions().untilAsserted(() -> { + assertFalse(pulsar1.getConfiguration().isLoadBalancerServiceUnitTableViewSyncerEnabled()); + primaryLoadManager.monitor(); + secondaryLoadManager.monitor(); + assertFalse(primaryLoadManager.getServiceUnitStateTableViewSyncer().isActive()); + assertFalse(secondaryLoadManager.getServiceUnitStateTableViewSyncer().isActive()); + }); + } } private void assertLookupHeartbeatOwner(PulsarService pulsar, @@ -1541,7 +1567,52 @@ private void makeSecondaryAsLeader() throws Exception { }); } - @Test(timeOut = 30 * 1000, priority = 2100) + // After a test churns leader election, the channel-topic bundle can be transiently + // unowned and the channel producer can be in reconnect backoff. The next @BeforeMethod + // (initializeState -> namespaces().unload(...)) publishes a state change on the channel + // topic; if it runs in that window the producer send times out (HTTP 500). Give the + // re-election a best-effort chance to settle before yielding to the next test. + // + // This is a best-effort smoothing wait, not an assertion: the budget is deliberately a + // fraction (20s) of the callers' 60s method timeout so it cannot consume the whole slot + // and trip TestNG's ThreadTimeoutException mid-poll, and any failure to settle + // is swallowed-and-logged rather than thrown. That matters because callers invoke this from + // a finally block — a settling delay here must never replace (mask) the body's exception. + // The next test's initializeState() carries a 60s ignoreExceptions retry as the real backstop. + private void awaitChannelOwnerStable() { + try { + Awaitility.await().atMost(20, TimeUnit.SECONDS).ignoreExceptions().untilAsserted(() -> { + // monitor() reconciles each broker's role with the channel ownership and + // re-serves the channel-topic bundle if the leadership churn left it unserved + // ("not served by this instance") — the same self-healing the 120s background + // monitor task provides, driven eagerly so the next test does not start inside + // the broken window. + primaryLoadManager.monitor(); + secondaryLoadManager.monitor(); + Optional owner1 = channel1.getChannelOwnerAsync().get(5, TimeUnit.SECONDS); + Optional owner2 = channel2.getChannelOwnerAsync().get(5, TimeUnit.SECONDS); + assertTrue(owner1.isPresent()); + assertEquals(owner1, owner2); + assertTrue(channel1.isChannelOwner() ^ channel2.isChannelOwner()); + // Probe that the channel topic is actually served: the lookup re-assigns the + // pulsar/system bundle if it is unowned, and getStats proves the owner loads + // the topic (the lookup layer alone can claim an owner that refuses to serve). + String channelTopic = ServiceUnitStateTableViewImpl.TOPIC; + assertNotNull(pulsar.getAdminClient().lookups().lookupTopic(channelTopic)); + if (serviceUnitStateTableViewClassName.equals( + ServiceUnitStateTableViewImpl.class.getName())) { + assertNotNull(pulsar.getAdminClient().topics().getStats(channelTopic)); + } + }); + } catch (Throwable t) { + log.warn("Channel owner did not stabilize within the best-effort window; " + + "relying on the next test's initializeState() retry", t); + } + } + + // 60s: the body's repeated role transitions plus the trailing awaitChannelOwnerStable() can + // exceed 30s under load. + @Test(timeOut = 60 * 1000, priority = 2100) public void testRoleChangeIdempotency() throws Exception { makePrimaryAsLeader(); @@ -1621,7 +1692,8 @@ public void testRoleChangeIdempotency() throws Exception { assertEquals(ExtensibleLoadManagerImpl.Role.Follower, secondaryLoadManager.getRole()); - + // Confirm a stable channel owner before yielding to the next test's @BeforeMethod. + awaitChannelOwnerStable(); } @DataProvider(name = "noChannelOwnerMonitorHandler") @@ -1629,7 +1701,9 @@ public Object[][] noChannelOwnerMonitorHandler() { return new Object[][] { { true }, { false } }; } - @Test(dataProvider = "noChannelOwnerMonitorHandler", timeOut = 30 * 1000, priority = 2101) + // 60s: the body's leader-election churn plus the trailing awaitChannelOwnerStable() can + // exceed 30s under load. + @Test(dataProvider = "noChannelOwnerMonitorHandler", timeOut = 60 * 1000, priority = 2101) public void testHandleNoChannelOwner(boolean noChannelOwnerMonitorHandler) throws Exception { makePrimaryAsLeader(); @@ -1696,10 +1770,25 @@ public void testHandleNoChannelOwner(boolean noChannelOwnerMonitorHandler) throw // clean up for monitor test pulsar1.getLeaderElectionService().start(); pulsar2.getLeaderElectionService().start(); + // If the body failed mid-churn, both restarted elections can keep flapping the + // leadership between the brokers, leaving the channel-topic bundle unserved + // beyond what the next test's initializeState() retry can absorb. Force a + // deterministic single leader before stabilizing (best-effort: must not mask + // the body's exception). + try { + makePrimaryAsLeader(); + } catch (Throwable t) { + log.warn("Failed to re-establish primary as leader in cleanup", t); + } + // Re-establish a stable channel owner before yielding to the next test's + // @BeforeMethod, which publishes to the channel topic via namespace unload. + awaitChannelOwnerStable(); } } - @Test(timeOut = 30 * 1000, priority = 2000) + // 60s: the body's role transitions plus the trailing awaitChannelOwnerStable() can exceed + // 30s under load (observed locally at 30.017s). + @Test(timeOut = 60 * 1000, priority = 2000) public void testRoleChange() throws Exception { makePrimaryAsLeader(); @@ -1719,6 +1808,11 @@ public void testRoleChange() throws Exception { new NamespaceBundleStats())); Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { + // The internal topics live in the pulsar/system bundle; if the leadership churn + // left it unserved, only a monitor() role reconciliation re-serves it (the + // background monitor task would take up to 120s) — drive it while waiting. + leader.monitor(); + follower.monitor(); assertNotNull(FieldUtils.readDeclaredField(leader.getTopBundlesLoadDataStore(), "tableView", true)); @@ -1756,6 +1850,9 @@ public void testRoleChange() throws Exception { topBundlesExpected.getTopBundlesLoadData().get(0).stats().msgRateIn = 1; Awaitility.await().atMost(30, TimeUnit.SECONDS).ignoreExceptions().untilAsserted(() -> { + // Same monitor()-driven healing as above for the post-transfer assertions. + leader2.monitor(); + follower2.monitor(); assertNotNull(FieldUtils.readDeclaredField(leader2.getTopBundlesLoadDataStore(), "tableView", true)); assertNull(FieldUtils.readDeclaredField(follower2.getTopBundlesLoadDataStore(), "tableView", true)); @@ -1780,6 +1877,9 @@ public void testRoleChange() throws Exception { follower2.getBrokerLoadDataStore().pushAsync(key, brokerLoadExpected).get(3, TimeUnit.SECONDS); follower2.getTopBundlesLoadDataStore().pushAsync(bundle, topBundlesExpected) .get(3, TimeUnit.SECONDS); + + // Confirm a stable channel owner before yielding to the next test's @BeforeMethod. + awaitChannelOwnerStable(); } @Test(priority = Integer.MIN_VALUE) From e5254fd66b050edae8400bbf77758cdc6732e318 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Mon, 8 Jun 2026 12:32:44 -0700 Subject: [PATCH 046/213] [fix][test] Fix flaky ExtensibleLoadManagerImplTest by re-serving the channel topic in initializeState (#25976) --- .../ExtensibleLoadManagerImplBaseTest.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplBaseTest.java index 2b30723f0a26a..ea74bd86127ec 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplBaseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplBaseTest.java @@ -199,16 +199,24 @@ protected void cleanup() throws Exception { protected void initializeState() throws PulsarAdminException, IllegalAccessException { // After a prior test churned leader election, the channel-topic bundle can be left // unserved ("not served by this instance"), making the unload's channel publish fail - // (HTTP 500) or hang server-side until the background monitor task (120s interval) - // reconciles the brokers' roles with the channel ownership. Drive monitor() eagerly to - // heal that state, bound each unload attempt (a synchronous unload() can block longer - // than the whole retry window), and fail loudly on exhaustion. + // (HTTP 500) or hang server-side. monitor() only self-heals when there is *no* channel + // owner; it does NOT heal the case where an owner is recorded but the bundle is not + // actually served, so the unload below can never publish. Force-serve the channel topic + // each attempt: an admin lookup re-assigns the pulsar/system bundle and getStats makes + // the owner load the topic (the lookup layer alone can claim an owner that refuses to + // serve). Bound each unload attempt and fail loudly on exhaustion. + boolean systemTopicChannel = + serviceUnitStateTableViewClassName.equals(ServiceUnitStateTableViewImpl.class.getName()); Awaitility.await().atMost(120, TimeUnit.SECONDS) .pollInterval(1, TimeUnit.SECONDS) .ignoreExceptions() .untilAsserted(() -> { primaryLoadManager.monitor(); secondaryLoadManager.monitor(); + if (systemTopicChannel) { + admin.lookups().lookupTopic(ServiceUnitStateTableViewImpl.TOPIC); + admin.topics().getStats(ServiceUnitStateTableViewImpl.TOPIC); + } admin.namespaces().unloadAsync(defaultTestNamespace).get(15, TimeUnit.SECONDS); }); reset(primaryLoadManager, secondaryLoadManager); From 0b27005cba4ccbaf6d7b54a4e4a28dde5a301759 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 8 Jun 2026 23:41:13 +0300 Subject: [PATCH 047/213] [fix][test] Fix flaky ExtensibleLoadManagerImplTest.initializeState by recovering wedged channel ownership (#25977) --- .../ExtensibleLoadManagerImplBaseTest.java | 77 +++++++++++++++---- .../ExtensibleLoadManagerImplTest.java | 3 +- 2 files changed, 64 insertions(+), 16 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplBaseTest.java index ea74bd86127ec..afd8681e24d4b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplBaseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplBaseTest.java @@ -20,6 +20,7 @@ import static org.mockito.Mockito.reset; import static org.mockito.Mockito.spy; +import static org.testng.Assert.assertTrue; import com.google.common.collect.Sets; import com.google.common.io.Resources; import java.util.ArrayList; @@ -38,7 +39,6 @@ import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateTableViewImpl; import org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder; import org.apache.pulsar.broker.testcontext.PulsarTestContext; -import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.impl.LookupService; @@ -48,6 +48,7 @@ import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; @@ -196,18 +197,39 @@ protected void cleanup() throws Exception { } @BeforeMethod(alwaysRun = true) - protected void initializeState() throws PulsarAdminException, IllegalAccessException { - // After a prior test churned leader election, the channel-topic bundle can be left - // unserved ("not served by this instance"), making the unload's channel publish fail - // (HTTP 500) or hang server-side. monitor() only self-heals when there is *no* channel - // owner; it does NOT heal the case where an owner is recorded but the bundle is not - // actually served, so the unload below can never publish. Force-serve the channel topic - // each attempt: an admin lookup re-assigns the pulsar/system bundle and getStats makes - // the owner load the topic (the lookup layer alone can claim an owner that refuses to - // serve). Bound each unload attempt and fail loudly on exhaustion. + protected void initializeState() throws Exception { + // Reset to a clean state before each test: reconcile each broker's role with the channel + // ownership and unload the test namespace so no bundle ownership carries over. The unload + // publishes a state change on the channel system topic. + // + // A prior role-churning test (e.g. the direct playLeader()/playFollower() calls in + // testRoleChangeIdempotency) can leave the channel system topic owned by a broker that no + // longer serves it ("not served by this instance, redo the lookup"), with the channel + // producer stuck in escalating reconnect backoff, so the unload's channel publish keeps + // failing. Each unload attempt force-serves the channel topic (an admin lookup re-assigns + // the pulsar/system bundle and getStats makes the owner load it); if that still does not + // recover, force a clean channel owner via leader re-election (which reassigns and + // re-serves the channel topic and makes clients redo their lookups) and retry. + try { + awaitTestNamespaceUnloaded(30); + } catch (ConditionTimeoutException channelWedged) { + recoverChannelOwnership(); + awaitTestNamespaceUnloaded(60); + } + reset(primaryLoadManager, secondaryLoadManager); + FieldUtils.writeDeclaredField(pulsarClient, "lookup", lookupService, true); + pulsar1.getConfig().setLoadBalancerMultiPhaseBundleUnload(true); + pulsar2.getConfig().setLoadBalancerMultiPhaseBundleUnload(true); + } + + // Drive monitor() to reconcile roles and force-serve the channel topic (monitor() only + // self-heals when there is *no* channel owner, not when an owner is recorded but the bundle is + // not served), then unload. ignoreExceptions() retries transient channel-publish failures; each + // unload attempt is bounded so a synchronous unload cannot block longer than the retry window. + private void awaitTestNamespaceUnloaded(long atMostSeconds) { boolean systemTopicChannel = serviceUnitStateTableViewClassName.equals(ServiceUnitStateTableViewImpl.class.getName()); - Awaitility.await().atMost(120, TimeUnit.SECONDS) + Awaitility.await().atMost(atMostSeconds, TimeUnit.SECONDS) .pollInterval(1, TimeUnit.SECONDS) .ignoreExceptions() .untilAsserted(() -> { @@ -219,10 +241,35 @@ protected void initializeState() throws PulsarAdminException, IllegalAccessExcep } admin.namespaces().unloadAsync(defaultTestNamespace).get(15, TimeUnit.SECONDS); }); - reset(primaryLoadManager, secondaryLoadManager); - FieldUtils.writeDeclaredField(pulsarClient, "lookup", lookupService, true); - pulsar1.getConfig().setLoadBalancerMultiPhaseBundleUnload(true); - pulsar2.getConfig().setLoadBalancerMultiPhaseBundleUnload(true); + } + + /** + * Force a clean channel owner via leader re-election. After heavy direct + * playLeader()/playFollower() churn the channel system topic can be left owned by a broker + * that no longer serves it, leaving the channel producer stuck on a stale lookup in + * escalating reconnect backoff. Closing the current owner's LeaderElectionService moves + * ownership to the other broker; its playLeader() re-creates and re-serves the channel + * topic, and the ownership change makes clients redo their (stale) lookups. + */ + private void recoverChannelOwnership() throws Exception { + boolean pulsar1Owns; + try { + pulsar1Owns = channel1.isChannelOwner(); + } catch (Exception e) { + // Owner can't be determined (e.g. no channel owner now); default to moving to pulsar2. + pulsar1Owns = true; + } + PulsarService currentOwner = pulsar1Owns ? pulsar1 : pulsar2; + ServiceUnitStateChannelImpl newOwnerChannel = pulsar1Owns ? channel2 : channel1; + currentOwner.getLeaderElectionService().close(); + try { + Awaitility.await().atMost(30, TimeUnit.SECONDS).ignoreExceptions() + .untilAsserted(() -> assertTrue(newOwnerChannel.isChannelOwner())); + } catch (ConditionTimeoutException ignore) { + // Best effort: the subsequent unload retry is the real backstop. + } finally { + currentOwner.getLeaderElectionService().start(); + } } protected void setPrimaryLoadManager() throws IllegalAccessException { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index 05e9bfba6efcf..24a62650125a5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -1578,7 +1578,8 @@ private void makeSecondaryAsLeader() throws Exception { // and trip TestNG's ThreadTimeoutException mid-poll, and any failure to settle // is swallowed-and-logged rather than thrown. That matters because callers invoke this from // a finally block — a settling delay here must never replace (mask) the body's exception. - // The next test's initializeState() carries a 60s ignoreExceptions retry as the real backstop. + // The next test's initializeState() is the real backstop: it retries the unload and, if the + // channel stays wedged, forces a clean channel owner via leader re-election before retrying. private void awaitChannelOwnerStable() { try { Awaitility.await().atMost(20, TimeUnit.SECONDS).ignoreExceptions().untilAsserted(() -> { From c2f68c688d738c94ac127f078c03b6a4383ee65b Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 8 Jun 2026 23:42:01 +0300 Subject: [PATCH 048/213] [fix][test] Deflake TopicPoliciesTest.setupTestTopic by retrying forced namespace deletion (#25974) --- .../pulsar/broker/admin/TopicPoliciesTest.java | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java index 0a1c76ff226ec..3621c9f21d557 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java @@ -175,16 +175,11 @@ void setupTestTopic() throws Exception { } catch (PulsarAdminException.NotFoundException e) { // topic may already be deleted } - try { - admin.namespaces().deleteNamespace(myNamespace, true); - } catch (PulsarAdminException.NotFoundException e) { - // namespace may already be deleted - } - try { - admin.namespaces().deleteNamespace(myNamespaceV1, true); - } catch (PulsarAdminException.NotFoundException e) { - // namespace may already be deleted - } + // Use deleteNamespaceWithRetry since the forced namespace deletion can fail transiently with HTTP 422 when a + // topic deletion in the cascade races with concurrent topic loading; the helper retries and treats an + // already-deleted namespace as success. + deleteNamespaceWithRetry(myNamespace, true); + deleteNamespaceWithRetry(myNamespaceV1, true); admin.namespaces().createNamespace(testTenant + "/" + testNamespace, Set.of("test")); admin.namespaces().createNamespace(myNamespaceV1); admin.topics().createPartitionedTopic(testTopic, testTopicPartitions); From 73dd67e81cd1721eea96944f79cd9be71769b086 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 8 Jun 2026 16:18:11 +0300 Subject: [PATCH 049/213] [improve][misc] Upgrade Apache Commons libraries and Apache Http components (#25963) (cherry picked from commit b686e96ae873ad59855e44312ff936e856c855a4) --- distribution/server/src/assemble/LICENSE.bin.txt | 14 +++++++------- distribution/shell/src/assemble/LICENSE.bin.txt | 6 +++--- pom.xml | 14 +++++++------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index f2cce7d0ca875..b2c117519b109 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -284,14 +284,14 @@ The Apache Software License, Version 2.0 * Apache Commons - commons-beanutils-commons-beanutils-1.11.0.jar - commons-cli-commons-cli-1.11.0.jar - - commons-codec-commons-codec-1.20.0.jar - - commons-io-commons-io-2.21.0.jar - - commons-logging-commons-logging-1.3.5.jar + - commons-codec-commons-codec-1.22.0.jar + - commons-io-commons-io-2.22.0.jar + - commons-logging-commons-logging-1.3.6.jar - org.apache.commons-commons-collections4-4.5.0.jar - org.apache.commons-commons-compress-1.28.0.jar - - org.apache.commons-commons-configuration2-2.15.0.jar + - org.apache.commons-commons-configuration2-2.15.1.jar - org.apache.commons-commons-lang3-3.19.0.jar - - org.apache.commons-commons-text-1.14.0.jar + - org.apache.commons-commons-text-1.15.0.jar * Netty - io.netty-netty-buffer-4.1.135.Final.jar - io.netty-netty-codec-4.1.135.Final.jar @@ -384,8 +384,8 @@ The Apache Software License, Version 2.0 - com.ascentstream.bookkeeper-native-io-4.17.4.0.jar - at.yawk.lz4-lz4-java-1.10.3.jar * Apache HTTP Client - - org.apache.httpcomponents-httpclient-4.5.13.jar - - org.apache.httpcomponents-httpcore-4.4.15.jar + - org.apache.httpcomponents-httpclient-4.5.14.jar + - org.apache.httpcomponents-httpcore-4.4.16.jar * AirCompressor - io.airlift-aircompressor-2.0.3.jar * AsyncHttpClient diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index efad8c79c253f..a465e9b3b4524 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -339,10 +339,10 @@ The Apache Software License, Version 2.0 - memory-0.8.3.jar - sketches-core-0.8.3.jar * Apache Commons - - commons-codec-1.20.0.jar - - commons-io-2.21.0.jar + - commons-codec-1.22.0.jar + - commons-io-2.22.0.jar - commons-lang3-3.19.0.jar - - commons-text-1.14.0.jar + - commons-text-1.15.0.jar - commons-compress-1.28.0.jar * Netty - netty-buffer-4.1.135.Final.jar diff --git a/pom.xml b/pom.xml index 623bcacc14c58..84e83368d4937 100644 --- a/pom.xml +++ b/pom.xml @@ -183,7 +183,7 @@ flexible messaging model and an intuitive client API. 4.17.4.0 3.9.5 1.11.0 - 1.14.0 + 1.15.0 1.1.10.8 4.1.12.1 5.7.1 @@ -267,9 +267,9 @@ flexible messaging model and an intuitive client API. 2.0.3 2.15.0 3.19.0 - 2.21.0 - 1.20.0 - 1.3.5 + 2.22.0 + 1.22.0 + 1.3.6 2.1.6 2.1.9 3.1.0 @@ -295,8 +295,8 @@ flexible messaging model and an intuitive client API. 1.0 9.1.6 6.2.12 - 4.5.13 - 4.4.15 + 4.5.14 + 4.4.16 0.7.7 0.7.2 2.0 @@ -374,7 +374,7 @@ flexible messaging model and an intuitive client API. 3.33.0 9.37.4 1.11.0 - 2.15.0 + 2.15.1 2.1.10 1.10.3 From 4b69157e8c28bb13c1de7b0c4e9cbb80e5002b24 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 12 Jun 2026 18:07:39 +0300 Subject: [PATCH 050/213] [fix][test][branch-4.0] Backport configurable read/add delays in PulsarMockBookKeeper Partial cherry-pick of the testmocks changes from 490ba0cca18 ([improve][broker] Implement PIP-430 Pulsar Broker cache improvements (#24623)), without the JFR read event interceptor parts which aren't needed on branch-4.0. Required so that CompactionTest compiles after cherry-picking ded1e42d352 (#25998), which uses PulsarMockBookKeeper.setDefaultReadEntriesDelayMillis. --- .../client/PulsarMockBookKeeper.java | 30 ++++++++++++++++ .../client/PulsarMockLedgerHandle.java | 36 +++++++++++-------- 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java index 7104ded746029..7bebc0e558f7a 100644 --- a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java +++ b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java @@ -75,6 +75,8 @@ public class PulsarMockBookKeeper extends BookKeeper { final OrderedExecutor orderedExecutor; final ExecutorService executor; final ScheduledExecutorService scheduler; + private volatile long defaultAddEntryDelayMillis = 1L; + private volatile long defaultReadEntriesDelayMillis = 1L; @Override public ClientConfiguration getConf() { @@ -492,5 +494,33 @@ public MetadataClientDriver getMetadataClientDriver() { return metadataClientDriver; } + public long getReadEntriesDelayMillis() { + return defaultReadEntriesDelayMillis; + } + + public long getNextAddEntryDelayMillis() { + Long delay = addEntryDelaysMillis.poll(); + if (delay != null) { + return delay; + } + return defaultAddEntryDelayMillis; + } + + public long getNextAddEntryResponseDelayMillis() { + Long delay = addEntryResponseDelaysMillis.poll(); + if (delay != null) { + return delay; + } + return 0; + } + + public void setDefaultAddEntryDelayMillis(long defaultAddEntryDelayMillis) { + this.defaultAddEntryDelayMillis = defaultAddEntryDelayMillis; + } + + public void setDefaultReadEntriesDelayMillis(long defaultReadEntriesDelayMillis) { + this.defaultReadEntriesDelayMillis = defaultReadEntriesDelayMillis; + } + private static final Logger log = LoggerFactory.getLogger(PulsarMockBookKeeper.class); } diff --git a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockLedgerHandle.java b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockLedgerHandle.java index 4d1fd1380c807..a540377413346 100644 --- a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockLedgerHandle.java +++ b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockLedgerHandle.java @@ -113,18 +113,26 @@ public void asyncClose(CloseCallback cb, Object ctx) { @Override public void asyncReadEntries(final long firstEntry, final long lastEntry, final ReadCallback cb, final Object ctx) { bk.getProgrammedFailure().thenComposeAsync((res) -> { - log.debug("readEntries: first={} last={} total={}", firstEntry, lastEntry, entries.size()); + if (log.isDebugEnabled()) { + log.debug("readEntries: first={} last={} total={}", firstEntry, lastEntry, entries.size()); + } final Queue seq = new ArrayDeque(); long entryId = firstEntry; while (entryId <= lastEntry && entryId < entries.size()) { seq.add(new LedgerEntry(entries.get((int) entryId++).duplicate())); } - log.debug("Entries read: {}", seq); + if (log.isDebugEnabled()) { + log.debug("Entries read: {}", seq); + } - try { - Thread.sleep(1); - } catch (InterruptedException e) { + long readEntriesDelay = bk.getReadEntriesDelayMillis(); + if (readEntriesDelay > 0) { + try { + Thread.sleep(readEntriesDelay); + } catch (InterruptedException e) { + // ignore + } } Enumeration entries = new Enumeration() { @@ -182,14 +190,12 @@ public void asyncAddEntry(final byte[] data, final int offset, final int length, @Override public void asyncAddEntry(final ByteBuf data, final AddCallback cb, final Object ctx) { bk.getAddEntryFailure().thenComposeAsync((res) -> { - Long delayMillis = bk.addEntryDelaysMillis.poll(); - if (delayMillis == null) { - delayMillis = 1L; - } - - try { - Thread.sleep(delayMillis); - } catch (InterruptedException e) { + long delayMillis = bk.getNextAddEntryDelayMillis(); + if (delayMillis > 0) { + try { + Thread.sleep(delayMillis); + } catch (InterruptedException e) { + } } if (fenced) { @@ -211,8 +217,8 @@ public void asyncAddEntry(final ByteBuf data, final AddCallback cb, final Object cb.addComplete(PulsarMockBookKeeper.getExceptionCode(exception), PulsarMockLedgerHandle.this, LedgerHandle.INVALID_ENTRY_ID, ctx); } else { - Long responseDelayMillis = bk.addEntryResponseDelaysMillis.poll(); - if (responseDelayMillis != null) { + long responseDelayMillis = bk.getNextAddEntryResponseDelayMillis(); + if (responseDelayMillis > 0) { try { Thread.sleep(responseDelayMillis); } catch (InterruptedException e) { From 59e61eb5e4679a3a1270cf5ab37e9b50c3a01b29 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 12 Jun 2026 11:34:33 +0800 Subject: [PATCH 051/213] [fix][broker] Fix compacted read could be stuck forever or message loss due to cursor mark delete (#25998) (cherry picked from commit 7a9fefb4c4a25b1aa2c0fb335d99aa7b00ee3c91) --- .../persistent/PersistentSubscription.java | 18 ++++ .../pulsar/compaction/CompactionTest.java | 85 +++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index 582af184b367b..9e5109f347752 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -58,6 +58,7 @@ import org.apache.pulsar.broker.intercept.BrokerInterceptor; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; +import org.apache.pulsar.broker.service.AbstractDispatcherSingleActiveConsumer; import org.apache.pulsar.broker.service.AbstractSubscription; import org.apache.pulsar.broker.service.AnalyzeBacklogResult; import org.apache.pulsar.broker.service.BrokerServiceException; @@ -443,6 +444,23 @@ public void acknowledgeMessage(List positions, AckType ackType, Map ml.getFirstPosition().getLedgerId()) { + log.warn("Received an ACK whose position is " + position + ", valid ledgers: " + + ml.getLedgersInfo().keySet()); + } + return; + } + } cursor.asyncMarkDelete(position, mergeCursorProperties(properties), markDeleteCallback, previousMarkDeletePosition); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java index c882fe460fd38..1d4856fe0536e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java @@ -56,6 +56,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.function.BiConsumer; @@ -70,12 +71,15 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerInfo; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.commons.lang3.mutable.MutableLong; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.intercept.MockBrokerInterceptor; import org.apache.pulsar.broker.namespace.NamespaceService; +import org.apache.pulsar.broker.service.ServerCnx; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; @@ -87,6 +91,7 @@ import org.apache.pulsar.client.api.EncryptionKeyInfo; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerBuilder; @@ -124,6 +129,7 @@ public class CompactionTest extends MockedPulsarServiceBaseTest { protected ScheduledExecutorService compactionScheduler; protected BookKeeper bk; private PublishingOrderCompactor compactor; + private volatile java.util.function.Consumer consumerCreated = __ -> {}; @Override protected void doInitConf() throws Exception { @@ -135,6 +141,14 @@ protected void doInitConf() throws Exception { @Override public void setup() throws Exception { super.internalSetup(); + pulsar.getBrokerService().setInterceptor(new MockBrokerInterceptor() { + + @Override + public void consumerCreated(ServerCnx cnx, org.apache.pulsar.broker.service.Consumer consumer, + Map metadata) { + consumerCreated.accept(consumer); + } + }); admin.clusters().createCluster(configClusterName, ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); @@ -164,6 +178,8 @@ public void beforeMethod() throws Exception { admin.namespaces().removeRetention("my-tenant/my-ns"); AbstractTwoPhaseCompactor.injectionAfterSeekInPhaseTwo = () -> {}; AbstractTwoPhaseCompactor.injectionPhaseTwoSeek = RawReader::seekAsync; + consumerCreated = __ -> {}; + pulsarTestContext.getMockBookKeeper().setDefaultReadEntriesDelayMillis(1); } protected long compact(String topic) throws ExecutionException, InterruptedException { @@ -2650,4 +2666,73 @@ private void triggerAndWaitCompaction(String topic) throws Exception { Awaitility.await().untilAsserted(() -> assertEquals( admin.topics().compactionStatus(topic).status, LongRunningProcessStatus.Status.SUCCESS)); } + + @Test + public void testReaderReadOnDeletedLedger() throws Exception { + final var topic = "persistent://my-tenant/my-ns/reader-read-on-deleted-ledger"; + try (final var producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create()) { + for (int i = 0; i < 3; i++) { + producer.newMessage().key("key-" + i).value("value-" + i).send(); + } + } + // Trigger the ledger rollover + var ml = (ManagedLedgerImpl) ((PersistentTopic) pulsar.getBrokerService().getTopicIfExists(topic).get() + .orElseThrow()).getManagedLedger(); + ml.getConfig().setMaxEntriesPerLedger(1); + ml.getConfig().setMaxSizePerLedgerMb(0); + ml.getConfig().setMinimumRolloverTime(0, TimeUnit.MILLISECONDS); + ml.rollCurrentLedgerIfFull(); + Awaitility.await().untilAsserted(() -> assertEquals(ml.getLedgersInfo().size(), 2)); + + final var subName = "sub-" + System.currentTimeMillis(); + @Cleanup final var reader = pulsarClient.newReader(Schema.STRING).readCompacted(true).topic(topic) + .subscriptionName(subName) + .startMessageId(MessageId.earliest).create(); + + // Slow down the pre-fetching + pulsarTestContext.getMockBookKeeper().setDefaultReadEntriesDelayMillis(500); + + // Receive 1 message so that the startMessageId will be reset to ledger_id:0 after reconnection + assertTrue(reader.hasMessageAvailable()); + final var firstMsg = reader.readNext(3, TimeUnit.SECONDS); + assertNotNull(firstMsg); + + triggerAndWaitCompaction(topic); + + // Simulate the pending cumulative acknowledgment is flushed after the consumer is created + // We don't need such interception if we can support controlling the acknowledgment flush for reader. + final var firstTime = new AtomicBoolean(true); + consumerCreated = serverConsumer -> { + final var subscription = serverConsumer.getSubscription(); + if (subscription.getName().contains(subName) && firstTime.compareAndSet(true, false)) { + final var msgId = (MessageIdAdv) firstMsg.getMessageId(); + subscription.acknowledgeMessage(List.of(PositionFactory.create(msgId.getLedgerId(), + msgId.getEntryId())), CommandAck.AckType.Cumulative, Map.of()); + } + }; + + // Trigger the reconnection and trim the first ledger. + admin.namespaces().unload("my-tenant/my-ns"); + admin.lookups().lookupTopic(topic); + final var persistentTopic = (PersistentTopic) pulsar.getBrokerService().getTopic(topic, true).get() + .orElseThrow(); + final var trimFuture = new CompletableFuture(); + persistentTopic.getManagedLedger().trimConsumedLedgersInBackground(trimFuture); + trimFuture.get(); + assertEquals(persistentTopic.getManagedLedger().getLedgersInfo().size(), 1); + + pulsarTestContext.getMockBookKeeper().setDefaultReadEntriesDelayMillis(1); + + while (reader.hasMessageAvailable()) { + final var msg = reader.readNextAsync().get(3, TimeUnit.SECONDS); + log.info("read id={} key={} value={}", msg.getMessageId(), msg.getKey(), msg.getValue()); + } + + final var serverConsumer = persistentTopic.getSubscription(subName).getDispatcher().getConsumers().get(0); + assertEquals(((MessageIdAdv) serverConsumer.getStartMessageId()).getEntryId(), 0L); + + final var emptyLedgerId = persistentTopic.getManagedLedger().getLedgersInfo().lastEntry().getKey(); + assertEquals(persistentTopic.getTopicCompactionService().getLastCompactedPosition().get(), + PositionFactory.create(emptyLedgerId, -1L)); + } } From 966c9572aca7c4b528d52fbdbb658237a56d843d Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 12 Jun 2026 08:40:43 +0300 Subject: [PATCH 052/213] [fix][meta] Keep the leader value in the election cycle and make leader reads authoritative (#26000) (cherry picked from commit c1f2a2b260728f00f0428db38ea5c57339163828) --- .../pulsar/broker/admin/impl/BrokersBase.java | 7 +- .../broker/admin/impl/NamespacesBase.java | 71 ++++---- .../loadbalance/LeaderElectionService.java | 10 + .../broker/namespace/NamespaceService.java | 131 +++++++------ .../LeaderElectionServiceTest.java | 4 + .../api/coordination/LeaderElection.java | 23 ++- .../coordination/impl/LeaderElectionImpl.java | 172 +++++++++++++----- .../replication/AutoRecoveryMainTest.java | 10 +- .../pulsar/metadata/LeaderElectionTest.java | 155 ++++++++++++++++ .../impl/LeaderElectionImplTest.java | 28 +++ 10 files changed, 464 insertions(+), 147 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java index 3ee2a1285d3b8..642acc7ac372b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java @@ -129,8 +129,11 @@ public void getActiveBrokers(@Suspended final AsyncResponse asyncResponse) throw public void getLeaderBroker(@Suspended final AsyncResponse asyncResponse) { validateBothSuperuserAndBrokerOperation(pulsar().getConfig().getClusterName(), pulsar().getBrokerId(), BrokerOperation.GET_LEADER_BROKER) - .thenAccept(__ -> { - LeaderBroker leaderBroker = pulsar().getLeaderElectionService().getCurrentLeader() + // The authoritative read: waits for an in-progress leader election to settle + // instead of returning 404 while a re-election is still in flight. + .thenCompose(__ -> pulsar().getLeaderElectionService().readCurrentLeader()) + .thenAccept(leader -> { + LeaderBroker leaderBroker = leader .orElseThrow(() -> new RestException(Status.NOT_FOUND, "Couldn't find leader broker")); BrokerInfo brokerInfo = BrokerInfo.builder() .serviceUrl(leaderBroker.getServiceUrl()) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 49784fc6b3eba..4235bc661c9d4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -57,7 +57,6 @@ import org.apache.commons.lang3.mutable.MutableObject; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.admin.AdminResource; -import org.apache.pulsar.broker.loadbalance.LeaderBroker; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException; @@ -1053,42 +1052,44 @@ private CompletableFuture validateLeaderBrokerAsync() { if (this.isLeaderBroker()) { return CompletableFuture.completedFuture(null); } - Optional currentLeaderOpt = pulsar().getLeaderElectionService().getCurrentLeader(); - if (currentLeaderOpt.isEmpty()) { - String errorStr = "The current leader is empty."; - log.error(errorStr); - return FutureUtil.failedFuture(new RestException(Response.Status.PRECONDITION_FAILED, errorStr)); - } - LeaderBroker leaderBroker = pulsar().getLeaderElectionService().getCurrentLeader().get(); - String leaderBrokerId = leaderBroker.getBrokerId(); - return pulsar().getNamespaceService() - .createLookupResult(leaderBrokerId, false, null) - .thenCompose(lookupResult -> { - String redirectUrl = isRequestHttps() ? lookupResult.getLookupData().getHttpUrlTls() - : lookupResult.getLookupData().getHttpUrl(); - if (redirectUrl == null) { - log.error("Redirected broker's service url is not configured"); - return FutureUtil.failedFuture(new RestException(Response.Status.PRECONDITION_FAILED, - "Redirected broker's service url is not configured.")); - } + // The authoritative read: waits for an in-progress leader election to settle instead of + // failing the request while a re-election is still in flight. + return pulsar().getLeaderElectionService().readCurrentLeader().thenCompose(currentLeaderOpt -> { + if (currentLeaderOpt.isEmpty()) { + String errorStr = "The current leader is empty."; + log.error(errorStr); + return FutureUtil.failedFuture(new RestException(Response.Status.PRECONDITION_FAILED, errorStr)); + } + String leaderBrokerId = currentLeaderOpt.get().getBrokerId(); + return pulsar().getNamespaceService() + .createLookupResult(leaderBrokerId, false, null) + .thenCompose(lookupResult -> { + String redirectUrl = isRequestHttps() ? lookupResult.getLookupData().getHttpUrlTls() + : lookupResult.getLookupData().getHttpUrl(); + if (redirectUrl == null) { + log.error("Redirected broker's service url is not configured"); + return FutureUtil.failedFuture(new RestException(Response.Status.PRECONDITION_FAILED, + "Redirected broker's service url is not configured.")); + } - try { - URL url = new URL(redirectUrl); - URI redirect = UriBuilder.fromUri(uri.getRequestUri()).host(url.getHost()) - .port(url.getPort()) - .replaceQueryParam("authoritative", - false).build(); - // Redirect - if (log.isDebugEnabled()) { - log.debug("Redirecting the request call to leader - {}", redirect); + try { + URL url = new URL(redirectUrl); + URI redirect = UriBuilder.fromUri(uri.getRequestUri()).host(url.getHost()) + .port(url.getPort()) + .replaceQueryParam("authoritative", + false).build(); + // Redirect + if (log.isDebugEnabled()) { + log.debug("Redirecting the request call to leader - {}", redirect); + } + return FutureUtil.failedFuture(( + new WebApplicationException(Response.temporaryRedirect(redirect).build()))); + } catch (MalformedURLException exception) { + log.error("The redirect url is malformed - {}", redirectUrl); + return FutureUtil.failedFuture(new RestException(exception)); } - return FutureUtil.failedFuture(( - new WebApplicationException(Response.temporaryRedirect(redirect).build()))); - } catch (MalformedURLException exception) { - log.error("The redirect url is malformed - {}", redirectUrl); - return FutureUtil.failedFuture(new RestException(exception)); - } - }); + }); + }); } public CompletableFuture setNamespaceBundleAffinityAsync(String bundleRange, String destinationBroker) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderElectionService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderElectionService.java index 2e53b54e98f61..21f67bd6b3613 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderElectionService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LeaderElectionService.java @@ -56,10 +56,20 @@ public void close() throws Exception { leaderElection.close(); } + /** + * Authoritative read of the current leader: if a leader election is in progress, the returned + * future completes once it settles (bounded by the default metadata operation timeout). Use + * this whenever a decision is made based on who the leader is. + */ public CompletableFuture> readCurrentLeader() { return leaderElection.getLeaderValue(); } + /** + * Non-blocking snapshot of the current leader; empty while a re-election is settling even + * though a leader may technically exist. Only suitable for best-effort uses such as logging — + * decision-making callers must use {@link #readCurrentLeader()}. + */ public Optional getCurrentLeader() { return leaderElection.getLeaderValueIfPresent(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index eeabb996c518c..8baa9bef67932 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -59,7 +59,6 @@ import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.broker.loadbalance.LeaderBroker; import org.apache.pulsar.broker.loadbalance.LeaderElectionService; import org.apache.pulsar.broker.loadbalance.LoadManager; import org.apache.pulsar.broker.loadbalance.ResourceUnit; @@ -571,7 +570,6 @@ public CompletableFuture getHeartbeatOrSLAMonitorBrokerId( private void searchForCandidateBroker(NamespaceBundle bundle, CompletableFuture> lookupFuture, LookupOptions options) { - String candidateBroker; LeaderElectionService les = pulsar.getLeaderElectionService(); if (les == null) { LOG.warn("The leader election has not yet been completed! NamespaceBundle[{}]", bundle); @@ -580,67 +578,98 @@ private void searchForCandidateBroker(NamespaceBundle bundle, return; } - boolean authoritativeRedirect = les.isLeader(); + selectCandidateBroker(bundle, options, les) + .thenAcceptAsync(selection -> { + if (selection.isEmpty()) { + LOG.warn("Load manager didn't return any available broker. " + + "Returning empty result to lookup. NamespaceBundle[{}]", + bundle); + lookupFuture.complete(Optional.empty()); + return; + } + acquireOwnershipOrRedirect(bundle, options, selection.get(), lookupFuture); + }, pulsar.getExecutor()) + .exceptionally(e -> { + LOG.warn("Error when searching for candidate broker to acquire {}: {}", bundle, e.getMessage(), e); + lookupFuture.completeExceptionally(FutureUtil.unwrapCompletionException(e)); + return null; + }); + } - try { - // check if this is Heartbeat or SLAMonitor namespace - candidateBroker = getHeartbeatOrSLAMonitorBrokerId(bundle, cb -> - CompletableFuture.completedFuture(isBrokerActive(cb))) - .get(config.getMetadataStoreOperationTimeoutSeconds(), SECONDS); + /** The broker selected for a bundle assignment, and whether the redirect to it is authoritative. */ + private record CandidateBrokerSelection(String candidateBroker, boolean authoritativeRedirect) { } - if (candidateBroker == null) { - Optional currentLeader = pulsar.getLeaderElectionService().getCurrentLeader(); + private CompletableFuture> selectCandidateBroker( + NamespaceBundle bundle, LookupOptions options, LeaderElectionService les) { + boolean authoritativeRedirect = les.isLeader(); - if (options.isAuthoritative()) { - // leader broker already assigned the current broker as owner - candidateBroker = pulsar.getBrokerId(); - } else { + // check if this is Heartbeat or SLAMonitor namespace + return getHeartbeatOrSLAMonitorBrokerId(bundle, cb -> + CompletableFuture.completedFuture(isBrokerActive(cb))) + .thenComposeAsync(heartbeatOrSlaBroker -> { + if (heartbeatOrSlaBroker != null) { + return completedSelection(heartbeatOrSlaBroker, authoritativeRedirect); + } + if (options.isAuthoritative()) { + // leader broker already assigned the current broker as owner + return completedSelection(pulsar.getBrokerId(), authoritativeRedirect); + } LoadManager loadManager = this.loadManager.get(); - boolean makeLoadManagerDecisionOnThisBroker = !loadManager.isCentralized() || les.isLeader(); - if (!makeLoadManagerDecisionOnThisBroker) { - // If leader is not active, fallback to pick the least loaded from current broker loadmanager + if (!loadManager.isCentralized() || les.isLeader()) { + return selectLeastLoadedBroker(bundle); + } + // The load manager decision belongs to the leader: read the leader + // authoritatively (waits for an in-progress election to settle) instead of + // acting on a possibly-empty snapshot during a leadership handoff. + return les.readCurrentLeader().thenComposeAsync(currentLeader -> { boolean leaderBrokerActive = currentLeader.isPresent() && isBrokerActive(currentLeader.get().getBrokerId()); - if (!leaderBrokerActive) { - makeLoadManagerDecisionOnThisBroker = true; - if (currentLeader.isEmpty()) { - LOG.warn( - "The information about the current leader broker wasn't available. " - + "Handling load manager decisions in a decentralized way. " - + "NamespaceBundle[{}]", - bundle); - } else { - LOG.warn( - "The current leader broker {} isn't active. " - + "Handling load manager decisions in a decentralized way. " - + "NamespaceBundle[{}]", - currentLeader.get(), bundle); - } + if (leaderBrokerActive) { + // forward to leader broker to make assignment + return completedSelection(currentLeader.get().getBrokerId(), authoritativeRedirect); } - } - if (makeLoadManagerDecisionOnThisBroker) { - Optional availableBroker = getLeastLoadedFromLoadManager(bundle); - if (availableBroker.isEmpty()) { - LOG.warn("Load manager didn't return any available broker. " - + "Returning empty result to lookup. NamespaceBundle[{}]", + // If leader is not active, fallback to pick the least loaded from current broker loadmanager + if (currentLeader.isEmpty()) { + LOG.warn( + "The information about the current leader broker wasn't available. " + + "Handling load manager decisions in a decentralized way. " + + "NamespaceBundle[{}]", bundle); - lookupFuture.complete(Optional.empty()); - return; + } else { + LOG.warn( + "The current leader broker {} isn't active. " + + "Handling load manager decisions in a decentralized way. " + + "NamespaceBundle[{}]", + currentLeader.get(), bundle); } - candidateBroker = availableBroker.get(); - authoritativeRedirect = true; - } else { - // forward to leader broker to make assignment - candidateBroker = currentLeader.get().getBrokerId(); - } - } - } + return selectLeastLoadedBroker(bundle); + }, pulsar.getExecutor()); + }, pulsar.getExecutor()); + } + + private static CompletableFuture> completedSelection( + String candidateBroker, boolean authoritativeRedirect) { + return CompletableFuture.completedFuture( + Optional.of(new CandidateBrokerSelection(candidateBroker, authoritativeRedirect))); + } + + // The decentralized decision is authoritative: this broker picked the owner itself. + private CompletableFuture> selectLeastLoadedBroker(NamespaceBundle bundle) { + Optional availableBroker; + try { + availableBroker = getLeastLoadedFromLoadManager(bundle); } catch (Exception e) { - LOG.warn("Error when searching for candidate broker to acquire {}: {}", bundle, e.getMessage(), e); - lookupFuture.completeExceptionally(e); - return; + return CompletableFuture.failedFuture(e); } + return CompletableFuture.completedFuture( + availableBroker.map(broker -> new CandidateBrokerSelection(broker, true))); + } + private void acquireOwnershipOrRedirect(NamespaceBundle bundle, LookupOptions options, + CandidateBrokerSelection selection, + CompletableFuture> lookupFuture) { + final String candidateBroker = selection.candidateBroker(); + final boolean authoritativeRedirect = selection.authoritativeRedirect(); try { Objects.requireNonNull(candidateBroker); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LeaderElectionServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LeaderElectionServiceTest.java index 3014c185cc802..a54a492144b99 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LeaderElectionServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LeaderElectionServiceTest.java @@ -21,6 +21,7 @@ import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import com.google.common.collect.Sets; import java.util.Optional; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; @@ -113,6 +114,9 @@ public void anErrorShouldBeThrowBeforeLeaderElected() throws PulsarServerExcepti leaderBrokerReference.get() != null); Mockito.when(leaderElectionService.getCurrentLeader()) .thenAnswer(invocation -> Optional.ofNullable(leaderBrokerReference.get())); + Mockito.when(leaderElectionService.readCurrentLeader()) + .thenAnswer(invocation -> + CompletableFuture.completedFuture(Optional.ofNullable(leaderBrokerReference.get()))); leaderElectionServiceReference.set(leaderElectionService); // broker, webService and leaderElectionService is started, but elect not ready; diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/coordination/LeaderElection.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/coordination/LeaderElection.java index 016b7d061d073..4b34815c558e6 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/coordination/LeaderElection.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/coordination/LeaderElection.java @@ -45,19 +45,30 @@ public interface LeaderElection extends AutoCloseable { LeaderElectionState getState(); /** - * Get the value set by the elected leader, or empty if there's currently no leader. + * Get the value set by the elected leader. + *

+ * This is the authoritative read: if a leader election is currently in progress (e.g. the + * previous leader's node was just deleted and the participants are re-electing), the returned + * future completes once the election has settled, with the newly determined leader value. The + * future completes exceptionally with a {@link java.util.concurrent.TimeoutException} if the + * election does not complete within the default metadata operation timeout. + *

+ * An instance that never participated in the election (no {@link #elect(Object)} call) reads + * the leader value directly from the metadata store. A closed instance does not wait: it + * reports an empty leader if it held the leadership when closed, or its last known view + * otherwise. * * @return a future that will track the completion of the operation */ CompletableFuture> getLeaderValue(); /** - * Get the value set by the elected leader, or empty if there's currently no leader. + * Get a non-blocking snapshot of the value set by the elected leader, or empty if no leader is + * known right now. *

- * The call is non blocking and in certain cases can return Optional.empty() even though a leader is - * technically elected. - * - * @return a future that will track the completion of the operation + * The snapshot can return Optional.empty() even though a leader is technically + * elected (for example while a re-election is still settling). Callers that need the + * authoritative leader must use {@link #getLeaderValue()} instead. */ Optional getLeaderValueIfPresent(); diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java index ab35eb7040c10..954bcfaf45a34 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java @@ -20,20 +20,18 @@ import com.fasterxml.jackson.databind.type.TypeFactory; import com.google.common.annotations.VisibleForTesting; +import java.time.Duration; import java.util.EnumSet; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.common.concurrent.FutureUtils; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.GetResult; -import org.apache.pulsar.metadata.api.MetadataCache; -import org.apache.pulsar.metadata.api.MetadataCacheConfig; import org.apache.pulsar.metadata.api.MetadataSerde; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.MetadataStoreException.AlreadyClosedException; @@ -52,14 +50,19 @@ class LeaderElectionImpl implements LeaderElection { private final String path; private final MetadataSerde serde; private final MetadataStoreExtended store; - private final MetadataCache cache; private final Consumer stateChangesListener; - private final ScheduledFuture updateCachedValueFuture; private LeaderElectionState leaderElectionState; private Optional version = Optional.empty(); private Optional proposedValue; + // The leader value as known by the election cycle (the leader can only change through an + // election cycle). Pending while no leader is known — election in progress or the leader node + // deleted — and completed with the leader value once the election settles. Readers of + // getLeaderValue() wait on it (bounded by leaderElectionCompletionTimeoutSeconds); + // getLeaderValueIfPresent() takes a non-blocking snapshot of it. + private CompletableFuture> currentLeaderFuture = new CompletableFuture<>(); + private final ScheduledExecutorService executor; private final FutureUtil.Sequencer sequencer; @@ -71,16 +74,21 @@ private enum InternalState { private static final int LEADER_ELECTION_RETRY_DELAY_SECONDS = 5; + // Upper bound for getLeaderValue() waiting on an election that never settles, aligned with the + // default metadata-store operation timeout (the broker's metadataStoreOperationTimeoutSeconds). + private volatile int leaderElectionCompletionTimeoutSeconds = 30; + + @VisibleForTesting + void setLeaderElectionCompletionTimeoutSeconds(int leaderElectionCompletionTimeoutSeconds) { + this.leaderElectionCompletionTimeoutSeconds = leaderElectionCompletionTimeoutSeconds; + } + LeaderElectionImpl(MetadataStoreExtended store, Class clazz, String path, Consumer stateChangesListener, ScheduledExecutorService executor) { this.path = path; this.serde = new JSONMetadataSerdeSimpleType<>(TypeFactory.defaultInstance().constructSimpleType(clazz, null)); this.store = store; - MetadataCacheConfig metadataCacheConfig = MetadataCacheConfig.builder() - .expireAfterWriteMillis(-1L) - .build(); - this.cache = store.getMetadataCache(clazz, metadataCacheConfig); this.leaderElectionState = LeaderElectionState.NoLeader; this.internalState = InternalState.Init; this.stateChangesListener = stateChangesListener; @@ -88,13 +96,38 @@ private enum InternalState { this.sequencer = FutureUtil.Sequencer.create(); store.registerListener(this::handlePathNotification); store.registerSessionListener(this::handleSessionNotification); - updateCachedValueFuture = executor.scheduleWithFixedDelay(this::getLeaderValue, - metadataCacheConfig.getRefreshAfterWriteMillis() / 2, - metadataCacheConfig.getRefreshAfterWriteMillis(), TimeUnit.MILLISECONDS); + } + + /** + * Record the leader value determined by the election cycle, waking up any getLeaderValue() + * callers waiting for the election to settle. + */ + private synchronized void leaderKnown(Optional leaderValue) { + if (currentLeaderFuture.isDone()) { + currentLeaderFuture = CompletableFuture.completedFuture(leaderValue); + } else { + currentLeaderFuture.complete(leaderValue); + } + } + + /** + * Mark the leader as unknown (the leader node was deleted) so getLeaderValue() callers wait for + * the next election cycle to settle instead of observing a stale value. + */ + private synchronized void leaderUnknown() { + if (currentLeaderFuture.isDone()) { + currentLeaderFuture = new CompletableFuture<>(); + } } @Override public synchronized CompletableFuture elect(T proposedValue) { + if (internalState == InternalState.Closed) { + // Reopened after close() (e.g. the broker's LeaderElectionService is close()d and then + // start()ed again): reset so a fresh election cycle runs and readers wait for it. + leaderElectionState = LeaderElectionState.NoLeader; + currentLeaderFuture = new CompletableFuture<>(); + } if (leaderElectionState != LeaderElectionState.NoLeader) { return CompletableFuture.completedFuture(leaderElectionState); } @@ -112,12 +145,6 @@ private synchronized CompletableFuture elect() { } else { return tryToBecomeLeader(); } - }).thenCompose(leaderElectionState -> { - // make sure that the cache contains the current leader - // so that getLeaderValueIfPresent works on all brokers - cache.refresh(path); - return cache.get(path) - .thenApply(__ -> leaderElectionState); }); } @@ -137,6 +164,7 @@ private synchronized CompletableFuture handleExistingLeader log.info("Keeping the existing value {} for {} as it's from the same session stat={}", existingValue, path, res.getStat()); // The value is still valid because it was created in the same session + leaderKnown(Optional.of(existingValue)); changeState(LeaderElectionState.Leading); return CompletableFuture.completedFuture(LeaderElectionState.Leading); } else { @@ -158,6 +186,7 @@ private synchronized CompletableFuture handleExistingLeader } // If the existing value is different, it means there's already another leader + leaderKnown(Optional.of(existingValue)); changeState(LeaderElectionState.Following); return CompletableFuture.completedFuture(LeaderElectionState.Following); } @@ -188,35 +217,18 @@ private synchronized CompletableFuture tryToBecomeLeader() .thenAccept(stat -> { synchronized (LeaderElectionImpl.this) { if (internalState == InternalState.ElectionInProgress) { - // Do a get() in order to force a notification later, if the z-node disappears - cache.get(path) - .thenRun(() -> { - synchronized (LeaderElectionImpl.this) { - log.info("Acquired leadership on {} with {}", path, value); - internalState = InternalState.LeaderIsPresent; - if (leaderElectionState != LeaderElectionState.Leading) { - leaderElectionState = LeaderElectionState.Leading; - try { - stateChangesListener.accept(leaderElectionState); - } catch (Throwable t) { - log.warn("Exception in state change listener", t); - } - } - result.complete(leaderElectionState); - } - }).exceptionally(ex -> { - // We fail to do the get(), so clean up the leader election fail the whole - // operation - log.warn("Failed to get the current state after acquiring leadership on {}. " - + " Conditionally deleting current entry.", path, ex); - store.delete(path, Optional.of(stat.getVersion())) - .thenRun(() -> result.completeExceptionally(ex)) - .exceptionally(ex2 -> { - result.completeExceptionally(ex2); - return null; - }); - return null; - }); + log.info("Acquired leadership on {} with {}", path, value); + internalState = InternalState.LeaderIsPresent; + leaderKnown(Optional.of(value)); + if (leaderElectionState != LeaderElectionState.Leading) { + leaderElectionState = LeaderElectionState.Leading; + try { + stateChangesListener.accept(leaderElectionState); + } catch (Throwable t) { + log.warn("Exception in state change listener", t); + } + } + result.complete(leaderElectionState); } else { log.info("Leadership on {} with value {} was lost. " + "Conditionally deleting entry with stat={}.", path, value, stat); @@ -254,7 +266,6 @@ private synchronized CompletableFuture tryToBecomeLeader() @Override public void close() throws Exception { - updateCachedValueFuture.cancel(true); try { asyncClose().join(); } catch (CompletionException e) { @@ -269,6 +280,12 @@ public synchronized CompletableFuture asyncClose() { } internalState = InternalState.Closed; + // A closed election reports "no leader" rather than waiting or failing: callers like the + // extensible load manager's handleNoChannelOwnerError() key off the resulting + // "no channel owner" condition to restart the election. + if (!currentLeaderFuture.isDone()) { + currentLeaderFuture.complete(Optional.empty()); + } if (leaderElectionState != LeaderElectionState.Leading) { return CompletableFuture.completedFuture(null); @@ -278,6 +295,9 @@ public synchronized CompletableFuture asyncClose() { .thenAccept(__ -> { synchronized (LeaderElectionImpl.this) { leaderElectionState = LeaderElectionState.NoLeader; + // The deleted leader node was ours and a closed instance no longer + // observes elections; don't keep reporting ourselves as leader. + currentLeaderFuture = CompletableFuture.completedFuture(Optional.empty()); } } ); @@ -290,12 +310,61 @@ public synchronized LeaderElectionState getState() { @Override public CompletableFuture> getLeaderValue() { - return cache.get(path); + CompletableFuture> future; + synchronized (this) { + if (internalState == InternalState.Init) { + // This instance never participated in the election (a pure observer, e.g. + // BookKeeper's MetadataDrivers helpers querying the current auditor): there is no + // local election cycle to wait for, so the store content is the authoritative + // answer. + return readLeaderValueFromStore(); + } + future = currentLeaderFuture; + } + if (future.isDone()) { + // Hand out a derived future so callers cannot complete the internal one. + return future.thenApply(value -> value); + } + int timeoutSeconds = leaderElectionCompletionTimeoutSeconds; + return FutureUtil.addTimeoutHandling(whenLeaderKnown(future), + Duration.ofSeconds(timeoutSeconds), executor, + () -> FutureUtil.createTimeoutException( + "Leader election on path " + path + " did not complete within " + + timeoutSeconds + " seconds", + LeaderElectionImpl.class, "getLeaderValue()")); + } + + private CompletableFuture> readLeaderValueFromStore() { + return store.get(path).thenApply(optRes -> optRes.map(res -> { + try { + return serde.deserialize(path, res.getValue(), res.getStat()); + } catch (Throwable t) { + throw new CompletionException(t); + } + })); + } + + // Track the internal future without exposing it: completing/cancelling the returned future + // (e.g. by the timeout handling) must not complete the election's own future. + private CompletableFuture> whenLeaderKnown(CompletableFuture> future) { + CompletableFuture> result = new CompletableFuture<>(); + future.whenComplete((value, ex) -> { + if (ex != null) { + result.completeExceptionally(ex); + } else { + result.complete(value); + } + }); + return result; } @Override public Optional getLeaderValueIfPresent() { - return cache.getIfCached(path); + CompletableFuture> future; + synchronized (this) { + future = currentLeaderFuture; + } + return future.isDone() && !future.isCompletedExceptionally() ? future.join() : Optional.empty(); } private void handleSessionNotification(SessionEvent event) { @@ -333,6 +402,9 @@ private void handlePathNotification(Notification notification) { } leaderElectionState = LeaderElectionState.NoLeader; + // The leader is unknown until the re-election below settles; getLeaderValue() + // callers wait for it instead of observing the stale value. + leaderUnknown(); if (proposedValue.isPresent()) { elect() diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java index 1d741c551ddb9..58ef6b6fcf990 100644 --- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java @@ -142,12 +142,16 @@ public void testAutoRecoverySessionLoss() throws Exception { } BookieId currentAuditor = main1.auditorElector.getCurrentAuditor(); assertNotNull(currentAuditor); - Auditor auditor1 = main1.auditorElector.getAuditor(); assertEquals("Current Auditor should be AR1", currentAuditor, BookieImpl.getBookieId(confByIndex(0))); + // getCurrentAuditor() can resolve as soon as the election settles, before the elector + // thread has constructed the Auditor instance — re-read getAuditor() on every poll instead + // of capturing a possibly-null reference once. Awaitility.waitAtMost(30, TimeUnit.SECONDS).untilAsserted(() -> { - assertNotNull(auditor1); - assertTrue("Auditor of AR1 should be running", auditor1.isRunning()); + Auditor a1 = main1.auditorElector.getAuditor(); + assertNotNull(a1); + assertTrue("Auditor of AR1 should be running", a1.isRunning()); }); + Auditor auditor1 = main1.auditorElector.getAuditor(); /* diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/LeaderElectionTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/LeaderElectionTest.java index 4b48f3c20b02b..cb26953738374 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/LeaderElectionTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/LeaderElectionTest.java @@ -18,10 +18,13 @@ */ package org.apache.pulsar.metadata; +import static org.assertj.core.api.Assertions.assertThat; import static org.testng.Assert.assertEquals; import java.util.EnumSet; +import java.util.List; import java.util.Optional; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; @@ -35,6 +38,7 @@ import org.apache.pulsar.metadata.api.extended.CreateOption; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.apache.pulsar.metadata.coordination.impl.CoordinationServiceImpl; +import org.awaitility.Awaitility; import org.testng.annotations.Test; public class LeaderElectionTest extends BaseMetadataStoreTest { @@ -284,4 +288,155 @@ public void revalidateLeaderWithDifferentSessionsDifferentValue(String provider, assertEquals(le.getLeaderValue().join(), Optional.of("test-1")); assertEqualsAndRetry(() -> le.getLeaderValueIfPresent(), Optional.of("test-1"), Optional.empty()); } + + @Test(dataProvider = "impl", timeOut = 30000) + public void readsDoNotObserveEmptyLeaderDuringReElection(String provider, Supplier urlSupplier) + throws Exception { + @Cleanup + MetadataStoreExtended store = MetadataStoreExtended.create(urlSupplier.get(), + MetadataStoreConfig.builder().fsyncEnable(false).build()); + + String path = newKey(); + + @Cleanup + CoordinationService cs = new CoordinationServiceImpl(store); + + @Cleanup + LeaderElection le = cs.getLeaderElection(String.class, path, __ -> { + }); + + assertEquals(le.elect("test-1").join(), LeaderElectionState.Leading); + + // Externally delete the leader node: the instance re-elects itself. An authoritative read + // issued during the churn either returns the last settled value or waits for the + // re-election to settle — it never observes an empty leader. + store.delete(path, Optional.empty()).join(); + assertEquals(le.getLeaderValue().join(), Optional.of("test-1")); + + Awaitility.await().untilAsserted(() -> { + assertEquals(le.getState(), LeaderElectionState.Leading); + assertEquals(le.getLeaderValueIfPresent(), Optional.of("test-1")); + }); + } + + @Test(dataProvider = "zkImpls", timeOut = 30000) + public void followerReadsResolveToTheNewLeaderAfterHandoff(String provider, Supplier urlSupplier) + throws Exception { + @Cleanup + MetadataStoreExtended store1 = MetadataStoreExtended.create(urlSupplier.get(), + MetadataStoreConfig.builder().build()); + @Cleanup + MetadataStoreExtended store2 = MetadataStoreExtended.create(urlSupplier.get(), + MetadataStoreConfig.builder().build()); + + String path = newKey(); + + @Cleanup + CoordinationService cs1 = new CoordinationServiceImpl(store1); + @Cleanup + CoordinationService cs2 = new CoordinationServiceImpl(store2); + + @Cleanup + LeaderElection le1 = cs1.getLeaderElection(String.class, path, __ -> { + }); + @Cleanup + LeaderElection le2 = cs2.getLeaderElection(String.class, path, __ -> { + }); + + assertEquals(le1.elect("test-1").join(), LeaderElectionState.Leading); + assertEquals(le2.elect("test-2").join(), LeaderElectionState.Following); + assertEquals(le2.getLeaderValue().join(), Optional.of("test-1")); + + // The leader hands off: le2 re-elects itself. Authoritative reads during the handoff + // return one of the settled leader values and converge to the new leader, but never + // observe an empty leader. + le1.close(); + List> observed = new CopyOnWriteArrayList<>(); + Awaitility.await().untilAsserted(() -> { + Optional leader = le2.getLeaderValue().join(); + observed.add(leader); + assertEquals(leader, Optional.of("test-2")); + }); + assertThat(observed) + .as("authoritative reads during the leadership handoff") + .doesNotContain(Optional.empty()); + } + + @Test(dataProvider = "impl", timeOut = 30000) + public void closedLeaderReportsEmptyLeader(String provider, Supplier urlSupplier) throws Exception { + @Cleanup + MetadataStoreExtended store = MetadataStoreExtended.create(urlSupplier.get(), + MetadataStoreConfig.builder().fsyncEnable(false).build()); + + String path = newKey(); + + @Cleanup + CoordinationService cs = new CoordinationServiceImpl(store); + + LeaderElection le = cs.getLeaderElection(String.class, path, __ -> { + }); + + assertEquals(le.elect("test-1").join(), LeaderElectionState.Leading); + assertEquals(le.getLeaderValue().join(), Optional.of("test-1")); + + // Closing the leader releases the leadership; reads on the closed instance must report an + // empty leader without waiting (recovery paths key off the "no leader" condition). + le.close(); + assertEquals(le.getLeaderValue().join(), Optional.empty()); + assertEquals(le.getLeaderValueIfPresent(), Optional.empty()); + } + + @Test(dataProvider = "impl", timeOut = 30000) + public void electAfterCloseRunsANewElection(String provider, Supplier urlSupplier) throws Exception { + @Cleanup + MetadataStoreExtended store = MetadataStoreExtended.create(urlSupplier.get(), + MetadataStoreConfig.builder().fsyncEnable(false).build()); + + String path = newKey(); + + @Cleanup + CoordinationService cs = new CoordinationServiceImpl(store); + + @Cleanup + LeaderElection le = cs.getLeaderElection(String.class, path, __ -> { + }); + + assertEquals(le.elect("test-1").join(), LeaderElectionState.Leading); + le.close(); + + // Re-electing on a closed instance reopens it (the broker's LeaderElectionService is + // close()d and start()ed again to force a leadership change). + assertEquals(le.elect("test-1").join(), LeaderElectionState.Leading); + assertEquals(le.getLeaderValue().join(), Optional.of("test-1")); + assertEquals(le.getLeaderValueIfPresent(), Optional.of("test-1")); + } + + @Test(dataProvider = "impl", timeOut = 30000) + public void observerReadsLeaderValueFromStore(String provider, Supplier urlSupplier) throws Exception { + @Cleanup + MetadataStoreExtended store = MetadataStoreExtended.create(urlSupplier.get(), + MetadataStoreConfig.builder().fsyncEnable(false).build()); + + String path = newKey(); + + @Cleanup + CoordinationService cs = new CoordinationServiceImpl(store); + @Cleanup + CoordinationService observerCs = new CoordinationServiceImpl(store); + + @Cleanup + LeaderElection le = cs.getLeaderElection(String.class, path, __ -> { + }); + // The observer never calls elect(): there is no local election cycle to wait for, so the + // authoritative read goes directly to the metadata store, while the snapshot stays empty. + @Cleanup + LeaderElection observer = observerCs.getLeaderElection(String.class, path, __ -> { + }); + + assertEquals(observer.getLeaderValue().join(), Optional.empty()); + + assertEquals(le.elect("test-1").join(), LeaderElectionState.Leading); + assertEquals(observer.getLeaderValue().join(), Optional.of("test-1")); + assertEquals(observer.getLeaderValueIfPresent(), Optional.empty()); + } } diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java index 09c9d71c41a30..4827c4a197ad5 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java @@ -18,7 +18,14 @@ */ package org.apache.pulsar.metadata.coordination.impl; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeoutException; import java.util.function.Supplier; import lombok.Cleanup; import org.apache.pulsar.metadata.BaseMetadataStoreTest; @@ -62,4 +69,25 @@ public void validateDeadLock(String provider, Supplier urlSupplier) }); blockFuture.join(); } + + @Test(timeOut = 20000) + public void getLeaderValueTimesOutWhenElectionNeverCompletes() { + MetadataStoreExtended store = mock(MetadataStoreExtended.class); + // The store never answers, so the election never settles. + when(store.get(anyString())).thenReturn(new CompletableFuture<>()); + + @Cleanup("shutdownNow") + ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + LeaderElectionImpl le = new LeaderElectionImpl<>(store, String.class, + "/getLeaderValueTimesOutWhenElectionNeverCompletes", __ -> { + }, executor); + le.setLeaderElectionCompletionTimeoutSeconds(1); + + le.elect("test-1"); + + assertThatThrownBy(() -> le.getLeaderValue().join()) + .hasCauseInstanceOf(TimeoutException.class) + .cause() + .hasMessageContaining("did not complete within"); + } } From 609cbfb7b13d4fe427aa1146eba95d21bb7d5a31 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 22 Jun 2026 14:21:11 +0300 Subject: [PATCH 053/213] [fix][build][branch-4.0] Upgrade docker/setup-qemu-action to v4.1.0 - previous approved hash was removed by https://github.com/apache/infrastructure-actions/commit/8d7b8628 --- .github/workflows/pulsar-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index b142b2ea416c7..73695b80e7d29 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -537,7 +537,7 @@ jobs: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries - name: Set up QEMU - uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 + uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 # v4.1.0 with: platforms: arm64 From 8ad9d8443cd14f66fd8f530e357591deddcd53ce Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Sat, 13 Jun 2026 04:55:20 +0300 Subject: [PATCH 054/213] [fix][broker] Prevent subscribe rate limit from stalling compaction and blocking forced deletion (#26015) (cherry picked from commit a1b5a0dbd8d67f3fcb5db4db53f73e7acd8c825c) --- .../service/persistent/PersistentTopic.java | 11 +++++-- .../pulsar/compaction/CompactionTest.java | 29 +++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 30289ee50f5a3..35934eedf0bb8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1017,8 +1017,15 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St new NamingException("Subscription with reserved subscription name attempted")); } - if (cnx.clientAddress() != null && cnx.clientAddress().toString().contains(":") - && subscribeRateLimiter.isPresent()) { + // The subscribe rate limit must not apply to the broker-internal compaction subscription: the + // compactor's reader re-subscribes after the phase-two seek, and throttling that re-subscribe stalls + // the compaction, which in turn blocks forced topic/namespace deletion waiting on the in-flight + // compaction. System topics are exempt as well, consistent with the publish/dispatch rate limiters, + // since throttling broker-internal readers (e.g. on __change_events) can stall topic policy updates. + if (subscribeRateLimiter.isPresent() + && !isSystemTopic() + && !isCompactionSubscription(subscriptionName) + && cnx.clientAddress() != null && cnx.clientAddress().toString().contains(":")) { SubscribeRateLimiter.ConsumerIdentifier consumer = new SubscribeRateLimiter.ConsumerIdentifier( cnx.clientAddress().toString().split(":")[0], consumerName, consumerId); if (!subscribeRateLimiter.get().subscribeAvailable(consumer) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java index 1d4856fe0536e..4adf3088b3571 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java @@ -110,6 +110,7 @@ import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; import org.apache.pulsar.common.policies.data.RetentionPolicies; +import org.apache.pulsar.common.policies.data.SubscribeRate; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.protocol.Markers; import org.apache.pulsar.common.util.FutureUtil; @@ -196,6 +197,34 @@ protected PublishingOrderCompactor getCompactor() { return compactor; } + @Test + public void testCompactionNotBlockedBySubscribeRateLimit() throws Exception { + String namespace = "my-tenant/my-ns"; + String topic = "persistent://" + namespace + "/compaction-with-subscribe-rate-limit"; + + try (Producer producer = pulsarClient.newProducer().topic(topic).enableBatching(false).create()) { + for (int i = 0; i < 10; i++) { + producer.newMessage().key("key" + (i % 2)).value(("my-message-" + i).getBytes()).send(); + } + } + + // Allow a single subscribe per consumer within a long period. The compactor's reader consumes the only + // token when it first subscribes, so the re-subscribe triggered by the phase-two seek would be throttled + // if the limit applied to the compaction subscription, stalling the compaction. + admin.namespaces().setSubscribeRate(namespace, new SubscribeRate(1, 3600)); + Awaitility.await().untilAsserted(() -> { + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic).get(); + assertTrue(persistentTopic.getSubscribeRateLimiter().isPresent()); + }); + + try { + compactor.compact(topic).get(30, TimeUnit.SECONDS); + } finally { + admin.namespaces().removeSubscribeRate(namespace); + } + } + @Test public void testCompaction() throws Exception { String topic = "persistent://my-tenant/my-ns/compaction"; From 4571d0d69396ab1804e3824e00e2a80df6382ccc Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Sat, 13 Jun 2026 08:53:57 +0300 Subject: [PATCH 055/213] [fix][broker] Fix forced topic/namespace deletion hanging or failing when compaction is in progress (#26016) (cherry picked from commit ec56aea8370e2dfdfda7aeeb1f5ee730d2d86866) --- .../service/persistent/PersistentTopic.java | 18 +++--- .../pulsar/client/impl/RawReaderImpl.java | 11 ++++ .../pulsar/client/impl/RawReaderTest.java | 19 ++++++ .../pulsar/compaction/CompactionTest.java | 58 +++++++++++++++++-- 4 files changed, 95 insertions(+), 11 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 35934eedf0bb8..2322ad9ca4a1f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1400,8 +1400,16 @@ private void asyncDeleteCursorWithCleanCompactionLedger(PersistentSubscription s return; } } - // Unsubscribe compaction cursor and delete compacted ledger. - currentCompaction.thenCompose(__ -> { + // Unsubscribe compaction cursor and delete compacted ledger. Wait for any in-flight compaction to finish + // first, but don't let a compaction that completed exceptionally block the cursor deletion: the deletion + // would otherwise fail on every retry until the topic instance is reloaded (issue #24148). Note that a + // fenced topic makes the compactor's reader fail with an unrecoverable error, so a forced deletion + // terminates an in-flight compaction exceptionally rather than waiting for it to complete normally. + currentCompaction.exceptionally(compactionEx -> { + log.info("[{}][{}] Last compaction task failed, proceeding to delete the compaction cursor", + topic, subscriptionName, compactionEx); + return null; + }).thenCompose(__ -> { asyncDeleteCursor(subscriptionName, unsubscribeFuture); return unsubscribeFuture; }).thenAccept(__ -> { @@ -1419,11 +1427,7 @@ private void asyncDeleteCursorWithCleanCompactionLedger(PersistentSubscription s disablingCompaction.compareAndSet(true, false); } }).exceptionally(ex -> { - if (currentCompaction.isCompletedExceptionally()) { - log.warn("[{}][{}] Last compaction task failed", topic, subscriptionName); - } else { - log.warn("[{}][{}] Failed to delete cursor task failed", topic, subscriptionName); - } + log.warn("[{}][{}] Failed to delete the compaction cursor", topic, subscriptionName, ex); // Reset the variable: disablingCompaction, disablingCompaction.compareAndSet(true, false); unsubscribeFuture.completeExceptionally(ex); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java index c3f8254fb5e30..1bc88ed0532da 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java @@ -230,6 +230,17 @@ CompletableFuture receiveRawAsync() { CompletableFuture result = new CompletableFuture<>(); pendingRawReceives.add(result); tryCompletePending(); + // Once the consumer has reached a terminal state (for example it was closed after an + // unrecoverable error such as the topic or namespace being deleted), no further message + // will arrive and no close callback will run for receives enqueued from now on, so the + // future would never complete. Re-checking the state after enqueueing closes the race + // with a concurrent close() draining the queue, since close() drains only after moving to + // a terminal state. This matters for compaction: a never-completing read leaves the + // compaction future pending, which in turn blocks forced topic/namespace deletion. + State state = getState(); + if (state == State.Closing || state == State.Closed || state == State.Failed) { + failPendingRawReceives(); + } return result; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java index bb98ca1e4e77d..9320e22e42e8b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java @@ -624,6 +624,25 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { admin.topics().delete(topic, false); } + @Test(timeOut = 30000) + public void testReadNextAsyncCompletesAfterConsumerClosed() throws Exception { + String topic = "persistent://my-property/my-ns/" + BrokerTestUtil.newUniqueName("reader"); + admin.topics().createNonPartitionedTopic(topic); + RawReader reader = RawReader.create(pulsarClient, topic, subscription).get(); + + // Put the reader's underlying consumer into a terminal state. In production this happens when a + // compaction's RawReader hits an unrecoverable error (e.g. the topic/namespace is being deleted). + reader.closeAsync().get(5, TimeUnit.SECONDS); + + // A read issued once the consumer has reached a terminal state must complete instead of hanging + // forever: a never-completing read keeps the compaction future pending, which blocks forced + // topic/namespace deletion (issue #24148). + CompletableFuture readFuture = reader.readNextAsync(); + Awaitility.await().atMost(10, TimeUnit.SECONDS) + .untilAsserted(() -> assertTrue(readFuture.isDone())); + assertTrue(readFuture.isCompletedExceptionally()); + } + @Test(timeOut = 100000) public void testPauseAndResume() throws Exception { log.info("-- Starting testPauseAndResume test --"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java index 4adf3088b3571..6ce5780f623e3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java @@ -136,6 +136,7 @@ public class CompactionTest extends MockedPulsarServiceBaseTest { protected void doInitConf() throws Exception { super.doInitConf(); conf.setDispatcherMaxReadBatchSize(1); + conf.setForceDeleteNamespaceAllowed(true); } @BeforeClass @@ -2540,10 +2541,59 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { Thread.sleep(3000); delayReadSignal.countDown(); - // Verify: topic deletion is successfully executed. - Awaitility.await().atMost(15, TimeUnit.SECONDS).untilAsserted(() -> { - assertTrue(deleteTopicFuture.isDone()); - }); + // Verify: topic deletion is successfully executed. Asserting success (not just completion) covers the + // case where fencing the topic terminates the in-flight compaction exceptionally: the failed compaction + // must not fail the deletion (issue #24148). + deleteTopicFuture.get(15, TimeUnit.SECONDS); + } + + @Test + public void testForcedDeleteSucceedsAfterFailedCompaction() throws Exception { + String topicName = newUniqueName("persistent://my-tenant/my-ns/delete-after-failed-compaction"); + admin.topics().createNonPartitionedTopic(topicName); + try (Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topicName).create()) { + for (int i = 0; i < 10; i++) { + producer.newMessage().key("key" + (i % 2)).value("value-" + i).send(); + } + } + + // Fail the compaction after the phase-two seek + AbstractTwoPhaseCompactor.injectionPhaseTwoSeek = + (reader, id) -> CompletableFuture.failedFuture(new RuntimeException("injected compaction failure")); + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false).join().get(); + persistentTopic.triggerCompaction(); + Awaitility.await().untilAsserted(() -> + assertEquals(persistentTopic.compactionStatus().status, LongRunningProcessStatus.Status.ERROR)); + AbstractTwoPhaseCompactor.injectionPhaseTwoSeek = RawReader::seekAsync; + + // The failed compaction must not block the deletion of the compaction cursor + admin.topics().delete(topicName, true); + } + + @Test + public void testForcedNamespaceDeleteWithInflightCompaction() throws Exception { + String namespace = "my-tenant/my-ns-inflight-compaction"; + admin.namespaces().createNamespace(namespace, Set.of(configClusterName)); + final String topicName = newUniqueName("persistent://" + namespace + "/inflight-compaction"); + admin.topics().createNonPartitionedTopic(topicName); + try (Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topicName).create()) { + // dispatcherMaxReadBatchSize=1 makes the compactor read these one at a time, keeping the compaction + // in-flight for several seconds while the namespace is deleted + for (int i = 0; i < 2000; i++) { + producer.newMessage().key(String.valueOf(i)).value(String.valueOf(i)).send(); + } + } + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false).join().get(); + persistentTopic.triggerCompaction(); + Awaitility.await().untilAsserted(() -> + assertEquals(persistentTopic.getSubscriptions().get(COMPACTION_SUBSCRIPTION).getConsumers().size(), + 1)); + + // Forced namespace deletion must succeed while the compaction is in-flight: fencing the topic terminates + // the compaction exceptionally, which must not fail the deletion of the compaction cursor (issue #24148) + deleteNamespaceWithRetry(namespace, true, admin); } @Test From ab890285d8314129c0521c18fa02c6f25a0703c3 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Sun, 14 Jun 2026 14:25:57 +0300 Subject: [PATCH 056/213] [fix][broker] Fix forced topic/namespace deletion still hanging when the compaction reader reconnect stalls (#26026) (cherry picked from commit 2325d1eb17d090afc0bd5eaeaa5c2a2e6a4445cf) --- .../service/persistent/PersistentTopic.java | 20 ++++-- .../pulsar/client/impl/RawReaderImpl.java | 20 ++++++ .../pulsar/client/impl/RawReaderTest.java | 63 +++++++++++++++++++ .../pulsar/compaction/CompactionTest.java | 37 +++++++++++ .../pulsar/client/impl/ConsumerImpl.java | 4 ++ 5 files changed, 138 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 2322ad9ca4a1f..e0bdbbc789c15 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1400,12 +1400,20 @@ private void asyncDeleteCursorWithCleanCompactionLedger(PersistentSubscription s return; } } - // Unsubscribe compaction cursor and delete compacted ledger. Wait for any in-flight compaction to finish - // first, but don't let a compaction that completed exceptionally block the cursor deletion: the deletion - // would otherwise fail on every retry until the topic instance is reloaded (issue #24148). Note that a - // fenced topic makes the compactor's reader fail with an unrecoverable error, so a forced deletion - // terminates an in-flight compaction exceptionally rather than waiting for it to complete normally. - currentCompaction.exceptionally(compactionEx -> { + // Unsubscribe compaction cursor and delete compacted ledger. Normally we wait for any in-flight compaction + // to finish first, but a compaction that completed exceptionally must not block the cursor deletion: it + // would otherwise fail on every retry until the topic instance is reloaded (issue #24148). + // + // Moreover, when the topic is being closed or deleted it is already fenced and any in-flight compaction is + // being aborted. Waiting for that compaction to complete is both unnecessary and unsafe here: the + // compactor's reader is expected to fail once the topic is fenced, but that depends on how the client + // reconnect surfaces the failure (a retriable lookup-stage error keeps the reader reconnecting instead of + // failing the in-flight read), so the compaction future may stay pending for far longer than the deletion + // can wait. Proceed with the cursor deletion right away in that case so a forced topic/namespace deletion + // cannot hang (issue #24148). + CompletableFuture compactionToWait = + isClosingOrDeleting ? CompletableFuture.completedFuture(null) : currentCompaction; + compactionToWait.exceptionally(compactionEx -> { log.info("[{}][{}] Last compaction task failed, proceeding to delete the compaction cursor", topic, subscriptionName, compactionEx); return null; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java index 1bc88ed0532da..940ec5dfe6011 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java @@ -41,6 +41,7 @@ import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.GrowableArrayBlockingQueue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -160,6 +161,25 @@ protected boolean isUnrecoverableError(Throwable t) { return super.isUnrecoverableError(t); } + @Override + public boolean connectionFailed(PulsarClientException exception) { + // A compaction reader is created with retryOnRecoverableErrors=false. When the topic is fenced or + // deleted, a reconnect can fail at the lookup/connection stage with a retriable error such as + // ServiceNotReadyException. The base ConsumerImpl.connectionFailed only consults isUnrecoverableError + // for non-retriable errors (or after the lookup deadline has passed), so for such a retriable error it + // would keep reconnecting and never fail the in-flight read, leaving the compaction future pending. + // Honor isUnrecoverableError here too so the reader is closed promptly and pending reads are failed, + // mirroring the subscribe-stage handling in ConsumerImpl.connectionOpened(). This matters for + // compaction: a never-failing read keeps the compaction future pending, which blocks forced + // topic/namespace deletion (issue #24148). + Throwable actError = FutureUtil.unwrapCompletionException(exception); + if (isUnrecoverableError(actError)) { + closeWhenReceivedUnrecoverableError(actError, null); + return false; + } + return super.connectionFailed(exception); + } + void tryCompletePending() { CompletableFuture future = null; RawMessageAndCnx messageAndCnx = null; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java index 9320e22e42e8b..a043f71923a6a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawReaderTest.java @@ -51,6 +51,7 @@ import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; @@ -643,6 +644,68 @@ public void testReadNextAsyncCompletesAfterConsumerClosed() throws Exception { assertTrue(readFuture.isCompletedExceptionally()); } + @Test(timeOut = 30000) + public void testConnectionFailureTerminatesReadWhenNotRetryingRecoverableErrors() throws Exception { + String topic = "persistent://my-property/my-ns/" + BrokerTestUtil.newUniqueName("reader"); + admin.topics().createNonPartitionedTopic(topic); + + // A compaction reader is created with retryOnRecoverableErrors=false. When the topic is fenced or deleted, + // a reconnect can fail at the lookup/connection stage (handled by ConsumerImpl.connectionFailed) with a + // retriable error such as ServiceNotReadyException. The base class would keep reconnecting, leaving the + // in-flight read pending forever; for a compaction reader that keeps the compaction future pending and + // blocks forced topic/namespace deletion (issue #24148). Such an unrecoverable error must instead + // terminate the reader and fail the in-flight read promptly. + ConsumerConfigurationData conf = new ConsumerConfigurationData<>(); + conf.getTopicNames().add(topic); + conf.setSubscriptionName(subscription); + conf.setSubscriptionType(SubscriptionType.Exclusive); + conf.setReceiverQueueSize(DEFAULT_RECEIVER_QUEUE_SIZE); + conf.setSubscriptionInitialPosition(SubscriptionInitialPosition.Earliest); + conf.setReadCompacted(true); + CompletableFuture> consumerFuture = new CompletableFuture<>(); + RawReaderImpl.RawConsumerImpl consumer = new RawReaderImpl.RawConsumerImpl( + (PulsarClientImpl) pulsarClient, conf, consumerFuture, false, false); + consumerFuture.get(10, TimeUnit.SECONDS); + + CompletableFuture readFuture = consumer.receiveRawAsync(); + boolean keepReconnecting = + consumer.connectionFailed(new PulsarClientException.ServiceNotReadyException("injected")); + + Assert.assertFalse(keepReconnecting); + Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> assertTrue(readFuture.isDone())); + assertTrue(readFuture.isCompletedExceptionally()); + } + + @Test(timeOut = 30000) + public void testConnectionFailureBeforeSubscribeFailsReaderCreation() throws Exception { + String topic = "persistent://my-property/my-ns/" + BrokerTestUtil.newUniqueName("reader"); + admin.topics().createNonPartitionedTopic(topic); + + // Same compaction-reader scenario as the test above, but the unrecoverable error (e.g. a lookup + // failure with ServiceNotReadyException) arrives BEFORE the initial subscribe completes. The + // subscribe (consumer) future must then be completed exceptionally; otherwise RawReader.create(...) + // would stay pending forever, which keeps the compaction future pending and blocks forced + // topic/namespace deletion (issue #24148). + ConsumerConfigurationData conf = new ConsumerConfigurationData<>(); + conf.getTopicNames().add(topic); + conf.setSubscriptionName(subscription); + conf.setSubscriptionType(SubscriptionType.Exclusive); + conf.setReceiverQueueSize(DEFAULT_RECEIVER_QUEUE_SIZE); + conf.setSubscriptionInitialPosition(SubscriptionInitialPosition.Earliest); + conf.setReadCompacted(true); + CompletableFuture> consumerFuture = new CompletableFuture<>(); + RawReaderImpl.RawConsumerImpl consumer = new RawReaderImpl.RawConsumerImpl( + (PulsarClientImpl) pulsarClient, conf, consumerFuture, false, false); + // Inject the failure without waiting for the subscribe to complete, i.e. while the consumer + // future is still pending (construction returns before the async subscribe round-trip finishes). + boolean keepReconnecting = + consumer.connectionFailed(new PulsarClientException.ServiceNotReadyException("injected")); + + Assert.assertFalse(keepReconnecting); + Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> assertTrue(consumerFuture.isDone())); + assertTrue(consumerFuture.isCompletedExceptionally()); + } + @Test(timeOut = 100000) public void testPauseAndResume() throws Exception { log.info("-- Starting testPauseAndResume test --"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java index 6ce5780f623e3..ac236c4f0770b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java @@ -2547,6 +2547,43 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { deleteTopicFuture.get(15, TimeUnit.SECONDS); } + @Test(timeOut = 60 * 1000) + public void testForcedDeleteCompletesWhileCompactionStuck() throws Exception { + final String topicName = newUniqueName("persistent://my-tenant/my-ns/forced-delete-stuck-compaction"); + admin.topics().createNonPartitionedTopic(topicName); + try (Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topicName).create()) { + for (int i = 0; i < 10; i++) { + producer.newMessage().key("key" + (i % 2)).value("value-" + i).send(); + } + } + + // Block the compaction at the phase-two seek so its future never completes on its own. This reproduces an + // in-flight compaction whose reader does not fail promptly when the topic is fenced (e.g. because a + // reconnect keeps retrying a retriable lookup-stage error): the compaction future stays pending (issue + // #24148). + CompletableFuture blockedSeek = new CompletableFuture<>(); + CountDownLatch reachedSeek = new CountDownLatch(1); + AbstractTwoPhaseCompactor.injectionPhaseTwoSeek = (reader, id) -> { + reachedSeek.countDown(); + return blockedSeek; + }; + try { + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false).join().get(); + persistentTopic.triggerCompaction(); + assertTrue(reachedSeek.await(30, TimeUnit.SECONDS)); + assertEquals(persistentTopic.compactionStatus().status, LongRunningProcessStatus.Status.RUNNING); + + // The compaction future is stuck, but a forced deletion must complete promptly instead of waiting for + // the in-flight compaction to finish (issue #24148). + persistentTopic.deleteForcefully().get(15, TimeUnit.SECONDS); + } finally { + AbstractTwoPhaseCompactor.injectionPhaseTwoSeek = RawReader::seekAsync; + // Unblock the stuck compaction so it can unwind and release its reader. + blockedSeek.complete(null); + } + } + @Test public void testForcedDeleteSucceedsAfterFailedCompaction() throws Exception { String topicName = newUniqueName("persistent://my-tenant/my-ns/delete-after-failed-compaction"); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 1038d84d89cf2..64c67dec0718b 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -1030,6 +1030,10 @@ protected void closeWhenReceivedUnrecoverableError(Throwable t, ClientCnx cnx) { final String cnxStr = cnx == null ? "null" : String.valueOf(cnx.channel().remoteAddress()); log.warn("[{}][{}] {} Closed consumer because get an error that does not support to retry: {} {}", topic, subscription, cnxStr, t.getClass().getName(), t.getMessage()); + // If the unrecoverable error occurs before the initial subscribe completes, fail the subscribe + // future as well; otherwise callers waiting on it (e.g. RawReader.create() / subscribeAsync()) + // would hang forever. This is a no-op when the subscribe future has already completed. + subscribeFuture.completeExceptionally(t); closeAsync().whenComplete((__, ex) -> { if (ex == null) { fail(t); From d703fd0aa544a3167b979fb2317a6514741dcab2 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Sun, 14 Jun 2026 16:22:58 +0300 Subject: [PATCH 057/213] [fix][broker] Don't let a stuck or aborted topic policies cache init make a namespace's topics unloadable (#26025) (cherry picked from commit 24f2270ec21906608429d05f86ec7347e9bf0edf) --- conf/broker.conf | 7 + conf/standalone.conf | 7 + .../pulsar/broker/ServiceConfiguration.java | 13 ++ .../SystemTopicBasedTopicPoliciesService.java | 135 +++++++++++++++++- ...temTopicBasedTopicPoliciesServiceTest.java | 121 ++++++++++++++-- 5 files changed, 271 insertions(+), 12 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index 503a7a0e11f5f..72a5f50876fa0 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -769,6 +769,13 @@ systemTopicSchemaCompatibilityStrategy=ALWAYS_COMPATIBLE # Please enable the system topic first. topicLevelPoliciesEnabled=true +# Amount of seconds to timeout initializing the topic policies cache of a namespace (reading the namespace's +# __change_events system topic to the end). Topic loading waits for this initialization, so if the system-topic +# reader gets stuck, this bounds the wait: the broker fails the initialization, closes the stuck reader and clears +# the cached state so loading the namespace's topics can be retried with a fresh reader instead of hanging until the +# broker is restarted. Set to 0 or a negative value to disable the timeout (not recommended). +topicPoliciesCacheInitTimeoutSeconds=60 + # If a topic remains fenced for this number of seconds, it will be closed forcefully. # If it is set to 0 or a negative number, the fenced topic will not be closed. topicFencingTimeoutSeconds=0 diff --git a/conf/standalone.conf b/conf/standalone.conf index f9c7ccb658f30..59fcbdf2aabc9 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -579,6 +579,13 @@ systemTopicSchemaCompatibilityStrategy=ALWAYS_COMPATIBLE # Please enable the system topic first. topicLevelPoliciesEnabled=true +# Amount of seconds to timeout initializing the topic policies cache of a namespace (reading the namespace's +# __change_events system topic to the end). Topic loading waits for this initialization, so if the system-topic +# reader gets stuck, this bounds the wait: the broker fails the initialization, closes the stuck reader and clears +# the cached state so loading the namespace's topics can be retried with a fresh reader instead of hanging until the +# broker is restarted. Set to 0 or a negative value to disable the timeout (not recommended). +topicPoliciesCacheInitTimeoutSeconds=60 + # If a topic remains fenced for this number of seconds, it will be closed forcefully. # If it is set to 0 or a negative number, the fenced topic will not be closed. topicFencingTimeoutSeconds=0 diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 132dfeb1358bf..80edc42cef0d6 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -552,6 +552,19 @@ The max allowed delay for delayed delivery (in milliseconds). If the broker rece ) private long topicLoadTimeoutSeconds = 60; + @FieldContext( + category = CATEGORY_SERVER, + dynamic = true, + doc = "Amount of seconds to timeout initializing the topic policies cache of a namespace (reading the " + + "namespace's __change_events system topic to the end). Topic loading waits for this " + + "initialization, so if the system-topic reader gets stuck (for example after __change_events is " + + "unloaded and the reconnected reader stops making progress), this bounds the wait: the broker " + + "fails the initialization, closes the stuck reader and clears the cached state so that loading " + + "the namespace's topics can be retried with a fresh reader instead of hanging until the broker " + + "is restarted. Set to 0 or a negative value to disable the timeout (not recommended)." + ) + private long topicPoliciesCacheInitTimeoutSeconds = 60; + @FieldContext( category = CATEGORY_SERVER, doc = "Whether we should enable metadata operations batching" diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index b47134f97b4e1..19aa0b4e8718d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -24,6 +24,7 @@ import com.github.benmanes.caffeine.cache.Caffeine; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Sets; +import io.opentelemetry.api.metrics.LongCounter; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -34,7 +35,9 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; @@ -80,10 +83,14 @@ */ public class SystemTopicBasedTopicPoliciesService implements TopicPoliciesService { + public static final String TOPIC_POLICIES_CACHE_INIT_TIMEOUT_METRIC_NAME = + "pulsar.broker.topic.policies.cache.init.timeout.count"; + private final PulsarService pulsarService; private final HashSet localCluster; private final String clusterName; private final AtomicBoolean closed = new AtomicBoolean(false); + private final LongCounter policyCacheInitTimeoutCounter; private final ConcurrentInitializer namespaceEventsSystemTopicFactoryLazyInitializer = new LazyInitializer<>() { @@ -127,6 +134,13 @@ public SystemTopicBasedTopicPoliciesService(PulsarService pulsarService) { this.pulsarService = pulsarService; this.clusterName = pulsarService.getConfiguration().getClusterName(); this.localCluster = Sets.newHashSet(clusterName); + this.policyCacheInitTimeoutCounter = pulsarService.getOpenTelemetry().getMeter() + .counterBuilder(TOPIC_POLICIES_CACHE_INIT_TIMEOUT_METRIC_NAME) + .setDescription("The number of times initializing a namespace's topic policies cache timed out " + + "because the __change_events system-topic reader was stuck. Each occurrence closes the " + + "stuck reader and clears the cached state so topic loading can be retried.") + .setUnit("{timeout}") + .build(); this.writerCaches = Caffeine.newBuilder() .expireAfterAccess(5, TimeUnit.MINUTES) .removalListener((namespaceName, writer, cause) -> { @@ -590,6 +604,12 @@ public void addOwnedNamespaceBundleAsync(NamespaceBundle namespaceBundle) { CompletableFuture existingFuture = policyCacheInitMap.putIfAbsent(namespace, initNamespacePolicyFuture); if (existingFuture == null) { + // Topic loading waits on this future, so a system-topic reader that gets stuck (e.g. after + // __change_events is unloaded and the reconnected reader stops making progress) would pin the + // policy cache for the whole namespace until the broker restarts (issue #25294). Bound the + // initialization so it fails fast and cleans up, letting topic loading retry with a fresh + // reader. + scheduleInitPolicesCacheTimeout(namespace, initNamespacePolicyFuture); final CompletableFuture> readerCompletableFuture = newReader(namespace); readerCompletableFuture @@ -604,14 +624,19 @@ public void addOwnedNamespaceBundleAsync(NamespaceBundle namespaceBundle) { return null; }).exceptionally(ex -> { try { + // Identity-guarded cleanup: a concurrent timeout cleanup or a namespace unload + // may already have dropped this future and let a retry install a fresh + // future/reader, so clean up by namespace key here would clobber that newer + // attempt. Tear down state only while this future still owns the namespace. if (readerCompletableFuture.isCompletedExceptionally()) { log.error("[{}] Failed to create reader on __change_events topic", namespace, ex); initNamespacePolicyFuture.completeExceptionally(ex); - cleanPoliciesCacheInitMap(namespace, true); + cleanupFailedPolicyCacheInit(namespace, initNamespacePolicyFuture, true); } else { initNamespacePolicyFuture.completeExceptionally(ex); - cleanPoliciesCacheInitMap(namespace, isAlreadyClosedException(ex)); + cleanupFailedPolicyCacheInit(namespace, initNamespacePolicyFuture, + isAlreadyClosedException(ex)); } } catch (Throwable cleanupEx) { // Adding this catch to avoid break callback chain @@ -692,6 +717,86 @@ public boolean test(NamespaceBundle namespaceBundle) { }); } + /** + * Bound the topic-policies cache initialization for {@code namespace} so a stuck {@code __change_events} reader + * cannot pin the cache (and therefore topic loading) for the whole namespace indefinitely (issue #25294). If the + * timeout wins the race to complete {@code initNamespacePolicyFuture}, the cached state is cleared and the stuck + * reader is closed so a subsequent load retries from scratch rather than waiting until the broker restarts. + */ + private void scheduleInitPolicesCacheTimeout(@NonNull NamespaceName namespace, + @NonNull CompletableFuture initNamespacePolicyFuture) { + long timeoutSeconds = pulsarService.getConfiguration().getTopicPoliciesCacheInitTimeoutSeconds(); + if (timeoutSeconds <= 0) { + return; + } + final ScheduledFuture timeoutTask = pulsarService.getExecutor().schedule(() -> { + TimeoutException timeoutException = new TimeoutException(String.format( + "Timed out after %d seconds initializing the topic policies cache for namespace %s; the " + + "__change_events reader did not reach the end of the topic", timeoutSeconds, namespace)); + if (initNamespacePolicyFuture.completeExceptionally(timeoutException)) { + policyCacheInitTimeoutCounter.add(1); + log.error("[{}] Timed out initializing the topic policies cache after {} seconds; closing the stuck " + + "__change_events reader so the namespace can be loaded again", namespace, timeoutSeconds); + try { + cleanupFailedPolicyCacheInit(namespace, initNamespacePolicyFuture, true); + } catch (Throwable cleanupEx) { + log.error("[{}] Failed to clean up the topic policies cache after init timeout", namespace, + cleanupEx); + } + } + }, timeoutSeconds, TimeUnit.SECONDS); + // Cancel the timeout once initialization finishes (successfully or not) so we don't leak scheduled tasks. + initNamespacePolicyFuture.whenComplete((__, ex) -> timeoutTask.cancel(false)); + } + + /** + * Identity-guarded cleanup for an initialization that failed (it timed out, the {@code __change_events} reader + * could not be created, or reading the topic threw). Unlike {@link #cleanPoliciesCacheInitMap}, which + * removes/closes by namespace key unconditionally, this only tears down state that still belongs to + * {@code initFuture}. By the time the failure is observed, a concurrent retry — or a namespace-bundle unload that + * left the init future orphaned — may already own the namespace with a fresh future and reader; removing by key + * would drop that newer future and close its reader, pinning the namespace again. Guarding on identity ensures a + * late failure never clobbers a newer initialization. + * + * @param closeReader when {@code true}, also clears the cached policies and closes the reader that belongs to this + * initialization; when {@code false}, only the init future is dropped, leaving the reader cached + * for the retry to reuse (mirrors the transient read-error path of + * {@link #cleanPoliciesCacheInitMap}). + */ + @VisibleForTesting + void cleanupFailedPolicyCacheInit(@NonNull NamespaceName namespace, + @NonNull CompletableFuture initFuture, boolean closeReader) { + // Capture the reader before dropping the init future so we only close the reader that belongs to this + // initialization, never one a concurrent retry creates immediately afterwards. + CompletableFuture> readerFuture = + closeReader ? readerCaches.get(namespace) : null; + if (!policyCacheInitMap.remove(namespace, initFuture)) { + // Superseded by a retry or an unload; that owner is responsible for its own reader/state. + return; + } + // Complete the dropped future (a no-op if the caller already completed it) outside any map remapping function, + // so awaiting topic loads fail fast and retry instead of hanging until the broker restarts (issue #25294). + failPendingPolicyCacheInit(namespace, initFuture); + if (!closeReader) { + return; + } + policiesCache.entrySet().removeIf(entry -> Objects.equals(entry.getKey().getNamespaceObject(), namespace)); + globalPoliciesCache.entrySet() + .removeIf(entry -> Objects.equals(entry.getKey().getNamespaceObject(), namespace)); + TopicPolicyMessageHandlerTracker tracker = topicPolicyMessageHandlerTrackers.remove(namespace); + if (tracker != null) { + tracker.close(); + } + if (readerFuture != null && readerCaches.remove(namespace, readerFuture) + && !readerFuture.isCompletedExceptionally()) { + readerFuture.thenCompose(SystemTopicClient.Reader::closeAsync) + .exceptionally(ex -> { + log.warn("[{}] Close change_event reader fail.", namespace, ex); + return null; + }); + } + } + private void initPolicesCache(SystemTopicClient.Reader reader, CompletableFuture future) { if (closed.get()) { future.completeExceptionally(new BrokerServiceException(getClass().getName() + " is closed.")); @@ -749,7 +854,7 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp @VisibleForTesting void cleanPoliciesCacheInitMap(@NonNull NamespaceName namespace, boolean closeReader) { if (!closeReader) { - policyCacheInitMap.remove(namespace); + failPendingPolicyCacheInit(namespace, policyCacheInitMap.remove(namespace)); return; } @@ -760,12 +865,18 @@ void cleanPoliciesCacheInitMap(@NonNull NamespaceName namespace, boolean closeRe } CompletableFuture> readerFuture = readerCaches.remove(namespace); + MutableObject> removedInitFuture = new MutableObject<>(); policyCacheInitMap.compute(namespace, (k, v) -> { + removedInitFuture.setValue(v); policiesCache.entrySet().removeIf(entry -> Objects.equals(entry.getKey().getNamespaceObject(), namespace)); globalPoliciesCache.entrySet() .removeIf(entry -> Objects.equals(entry.getKey().getNamespaceObject(), namespace)); return null; }); + // Complete the removed init future outside the compute() remapping function: completing it can run the + // awaiting topic-load callbacks synchronously, and doing that while holding the ConcurrentHashMap bin lock + // risks a recursive map update / deadlock (see #24977). + failPendingPolicyCacheInit(namespace, removedInitFuture.getValue()); if (readerFuture != null && !readerFuture.isCompletedExceptionally()) { readerFuture .thenCompose(SystemTopicClient.Reader::closeAsync) @@ -776,6 +887,20 @@ void cleanPoliciesCacheInitMap(@NonNull NamespaceName namespace, boolean closeRe } } + /** + * Complete an init future that is being dropped from {@link #policyCacheInitMap} but never completed, so the + * topic loads awaiting it fail fast and retry instead of hanging until the broker restarts (issue #25294). + * No-op if the future was already completed by its own initialization chain. + */ + private void failPendingPolicyCacheInit(@NonNull NamespaceName namespace, + @Nullable CompletableFuture initFuture) { + if (initFuture != null && !initFuture.isDone()) { + initFuture.completeExceptionally(new BrokerServiceException( + "Topic policies cache initialization for namespace " + namespace + + " was aborted because the cached state was cleared")); + } + } + private void cleanWriterCache(@NonNull NamespaceName namespace) { writerCaches.synchronous().invalidate(namespace); } @@ -1040,6 +1165,10 @@ public void close() throws Exception { } }); readerCaches.clear(); + // Release any topic loads still waiting on an in-progress policy cache initialization so they fail + // fast instead of hanging until the awaited future is completed indirectly (issue #25294). + policyCacheInitMap.forEach(this::failPendingPolicyCacheInit); + policyCacheInitMap.clear(); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java index 8d3b16723ed4f..1b5182a454526 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java @@ -28,6 +28,7 @@ import static org.testng.AssertJUnit.assertFalse; import static org.testng.AssertJUnit.assertNotNull; import static org.testng.AssertJUnit.assertNull; +import static org.testng.AssertJUnit.assertSame; import static org.testng.AssertJUnit.assertTrue; import java.time.Duration; import java.util.HashSet; @@ -41,12 +42,15 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.events.PulsarEvent; import org.apache.pulsar.common.naming.NamespaceName; @@ -554,17 +558,18 @@ public void testPrepareInitPoliciesCacheAsyncThrowExceptionAfterCreateReader() t Assert.assertNull(readerCompletableFuture1); }); - - // make sure not do cleanPoliciesCacheInitMap() twice - // totally trigger prepareInitPoliciesCacheAsync() twice, so the time of cleanPoliciesCacheInitMap() is 2. - // in previous code, the time would be 3 + // Cleanup must run exactly once per trigger and not repeat recursively (in older code it ran 3 times). + // Two failures are triggered here: the reader.close() above drives readMorePoliciesAsync into + // cleanPoliciesCacheInitMap (1x), and the second prepareInitPoliciesCacheAsync fails in initPolicesCache and + // is torn down by the identity-guarded cleanupFailedPolicyCacheInit (1x). boolean logFound = testLogAppender.getEvents().stream().anyMatch(logEvent -> logEvent.getMessage().toString().contains("Failed to create reader on __change_events topic")); assertFalse(logFound); boolean logFound2 = testLogAppender.getEvents().stream().anyMatch(logEvent -> logEvent.getMessage().toString().contains("Failed to check the move events for the system topic")); assertTrue(logFound2); - verify(spyService, times(2)).cleanPoliciesCacheInitMap(any(), anyBoolean()); + verify(spyService, times(1)).cleanPoliciesCacheInitMap(any(), anyBoolean()); + verify(spyService, times(1)).cleanupFailedPolicyCacheInit(any(), any(), anyBoolean()); // make sure not occur Recursive update boolean logFound3 = testLogAppender.getEvents().stream().anyMatch(logEvent -> @@ -623,9 +628,9 @@ public void testPrepareInitPoliciesCacheAsyncThrowExceptionInCreateReader() thro Assert.assertNull(readerCompletableFuture1); }); - - // make sure not do cleanPoliciesCacheInitMap() twice - // totally trigger prepareInitPoliciesCacheAsync() once, so the time of cleanPoliciesCacheInitMap() is 1. + // Reader creation fails, so the single cleanup runs once via the identity-guarded + // cleanupFailedPolicyCacheInit (the reader-creation-failure branch no longer goes through + // cleanPoliciesCacheInitMap), and must not run more than once. boolean logFound = testLogAppender.getEvents().stream().anyMatch(logEvent -> logEvent.getMessage().toString().contains("Failed to create reader on __change_events topic")); assertTrue(logFound); @@ -633,6 +638,104 @@ public void testPrepareInitPoliciesCacheAsyncThrowExceptionInCreateReader() thro logEvent.getMessage().toString().contains("Failed to check the move events for the system topic") || logEvent.getMessage().toString().contains("Failed to read event from the system topic")); assertFalse(logFound2); - verify(spyService, times(1)).cleanPoliciesCacheInitMap(any(), anyBoolean()); + verify(spyService, times(1)).cleanupFailedPolicyCacheInit(any(), any(), anyBoolean()); + verify(spyService, times(0)).cleanPoliciesCacheInitMap(any(), anyBoolean()); + } + + @Test(timeOut = 60_000) + public void testPrepareInitPoliciesCacheAsyncTimesOutWhenReaderStuck() throws Exception { + // Bound the policy-cache initialization to a short timeout for the test. + pulsar.getConfiguration().setTopicPoliciesCacheInitTimeoutSeconds(3); + + pulsar.getTopicPoliciesService().close(); + SystemTopicBasedTopicPoliciesService spyService = + Mockito.spy(new SystemTopicBasedTopicPoliciesService(pulsar)); + FieldUtils.writeField(pulsar, "topicPoliciesService", spyService, true); + + admin.namespaces().createNamespace(NAMESPACE5); + final NamespaceName namespace = NamespaceName.get(NAMESPACE5); + + // Create a real __change_events reader, then spy it so that it reports more events but never delivers one — + // i.e. a reader that reconnected but is stuck (issue #25294). initPolicesCache would otherwise never complete. + SystemTopicClient.Reader stuckReader = Mockito.spy(spyService.createSystemTopicClient(namespace) + .get(30, TimeUnit.SECONDS)); + Mockito.doReturn(CompletableFuture.completedFuture(true)).when(stuckReader).hasMoreEventsAsync(); + Mockito.doReturn(new CompletableFuture>()).when(stuckReader).readNextAsync(); + Mockito.doReturn(CompletableFuture.completedFuture(stuckReader)) + .when(spyService).createSystemTopicClient(namespace); + + // Without the timeout the returned future never completes and topic loading for the namespace hangs forever. + CompletableFuture prepareFuture = spyService.prepareInitPoliciesCacheAsync(namespace); + try { + prepareFuture.get(15, TimeUnit.SECONDS); + Assert.fail("Expected the topic policies cache initialization to time out"); + } catch (ExecutionException e) { + assertTrue("Expected a TimeoutException cause but got " + e.getCause(), + e.getCause() instanceof TimeoutException); + } + + // The poisoned cache entry must be cleared and the stuck reader closed so a subsequent load can retry with a + // fresh reader instead of being pinned until the broker restarts. + Awaitility.await().untilAsserted(() -> assertNull(spyService.getPoliciesCacheInit(namespace))); + Mockito.verify(stuckReader, Mockito.atLeastOnce()).closeAsync(); + } + + @Test + public void testCleanPoliciesCacheInitMapCompletesPendingInitFuture() { + SystemTopicBasedTopicPoliciesService service = + (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); + final NamespaceName namespace = NamespaceName.get(NAMESPACE1); + + // Dropping the cached init future (e.g. on a namespace-bundle unload) must complete it so the topic loads + // awaiting it fail fast and retry, instead of hanging until the broker restarts (issue #25294). + CompletableFuture pendingWithReaderClose = new CompletableFuture<>(); + service.policyCacheInitMap.put(namespace, pendingWithReaderClose); + service.cleanPoliciesCacheInitMap(namespace, true); + assertTrue(pendingWithReaderClose.isCompletedExceptionally()); + assertNull(service.getPoliciesCacheInit(namespace)); + + CompletableFuture pendingWithoutReaderClose = new CompletableFuture<>(); + service.policyCacheInitMap.put(namespace, pendingWithoutReaderClose); + service.cleanPoliciesCacheInitMap(namespace, false); + assertTrue(pendingWithoutReaderClose.isCompletedExceptionally()); + assertNull(service.getPoliciesCacheInit(namespace)); + + // An already-completed init future must not be overwritten/disturbed. + CompletableFuture alreadyDone = CompletableFuture.completedFuture(null); + service.policyCacheInitMap.put(namespace, alreadyDone); + service.cleanPoliciesCacheInitMap(namespace, true); + assertFalse(alreadyDone.isCompletedExceptionally()); + } + + @Test + @SuppressWarnings("unchecked") + public void testCleanupFailedPolicyCacheInitIsIdentityGuarded() { + SystemTopicBasedTopicPoliciesService service = + (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); + final NamespaceName namespace = NamespaceName.get(NAMESPACE1); + + // A newer init attempt (B) already owns the namespace with a fresh future and reader. + CompletableFuture newerInitFuture = new CompletableFuture<>(); + SystemTopicClient.Reader newerReader = Mockito.mock(SystemTopicClient.Reader.class); + Mockito.doReturn(CompletableFuture.completedFuture(null)).when(newerReader).closeAsync(); + service.policyCacheInitMap.put(namespace, newerInitFuture); + service.getReaderCaches().put(namespace, CompletableFuture.completedFuture(newerReader)); + + // A stale init attempt (A) whose reader was already torn down (e.g. by the timeout cleanup) fires its failure + // callback late. Cleaning it up by identity must be a no-op: it must not clobber B's future or close B's reader + // (issue #25294 follow-up). + CompletableFuture staleInitFuture = new CompletableFuture<>(); + service.cleanupFailedPolicyCacheInit(namespace, staleInitFuture, true); + assertSame(newerInitFuture, service.getPoliciesCacheInit(namespace)); + assertFalse(newerInitFuture.isDone()); + assertNotNull(service.getReaderCaches().get(namespace)); + Mockito.verify(newerReader, Mockito.never()).closeAsync(); + + // Cleaning up the owning attempt does drop its future and close its reader. + service.cleanupFailedPolicyCacheInit(namespace, newerInitFuture, true); + assertNull(service.getPoliciesCacheInit(namespace)); + assertTrue(newerInitFuture.isCompletedExceptionally()); + assertNull(service.getReaderCaches().get(namespace)); + Mockito.verify(newerReader, Mockito.times(1)).closeAsync(); } } From 195e431ab6547e0513cf7775acdf5a72fa3d62f6 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Mon, 15 Jun 2026 18:59:02 +0800 Subject: [PATCH 058/213] [fix][broker] Fail fast for load balancer misconfigurations instead of falling back to SimpleLoadManagerImpl (#26031) (cherry picked from commit c3f8c05e4f27b194d889cbe3eefc4d83548628aa) --- .../broker/loadbalance/LoadManager.java | 52 +++++++++---------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadManager.java index 9399fcb3aad7a..c9aa673d6b53d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadManager.java @@ -151,35 +151,31 @@ default void writeLoadReportOnZookeeper(boolean force) throws Exception { void initialize(PulsarService pulsar); static LoadManager create(final PulsarService pulsar) { - try { - final ServiceConfiguration conf = pulsar.getConfiguration(); - - String loadManagerClassName = conf.getLoadManagerClassName(); - if (StringUtils.isBlank(loadManagerClassName)) { - loadManagerClassName = SimpleLoadManagerImpl.class.getName(); - } - - // Assume there is a constructor with one argument of PulsarService. - final Object loadManagerInstance = Reflections.createInstance(loadManagerClassName, - Thread.currentThread().getContextClassLoader()); - if (loadManagerInstance instanceof LoadManager casted) { - casted.initialize(pulsar); - return casted; - } else if (loadManagerInstance instanceof ModularLoadManager modularLoadManager) { - final LoadManager casted = new ModularLoadManagerWrapper(modularLoadManager); - casted.initialize(pulsar); - return casted; - } else if (loadManagerInstance instanceof ExtensibleLoadManager) { - final LoadManager casted = - new ExtensibleLoadManagerWrapper((ExtensibleLoadManagerImpl) loadManagerInstance); - casted.initialize(pulsar); - return casted; - } - } catch (Exception e) { - LOG.warn("Error when trying to create load manager: ", e); + final ServiceConfiguration conf = pulsar.getConfiguration(); + + String loadManagerClassName = conf.getLoadManagerClassName(); + if (StringUtils.isBlank(loadManagerClassName)) { + loadManagerClassName = SimpleLoadManagerImpl.class.getName(); + } + + // Assume there is a constructor with one argument of PulsarService. + final Object loadManagerInstance = Reflections.createInstance(loadManagerClassName, + Thread.currentThread().getContextClassLoader()); + if (loadManagerInstance instanceof LoadManager casted) { + casted.initialize(pulsar); + return casted; + } else if (loadManagerInstance instanceof ModularLoadManager modularLoadManager) { + final LoadManager casted = new ModularLoadManagerWrapper(modularLoadManager); + casted.initialize(pulsar); + return casted; + } else if (loadManagerInstance instanceof ExtensibleLoadManager) { + final LoadManager casted = + new ExtensibleLoadManagerWrapper((ExtensibleLoadManagerImpl) loadManagerInstance); + casted.initialize(pulsar); + return casted; + } else { + throw new IllegalArgumentException(loadManagerClassName + " is not a supported LoadManager"); } - // If we failed to create a load manager, default to SimpleLoadManagerImpl. - return new SimpleLoadManagerImpl(pulsar); } } From d5ba72bb3198f5793d2d857af839eb1fd692bbfe Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 16 Jun 2026 11:23:00 +0300 Subject: [PATCH 059/213] [improve][fn] Upgrade pulsar-client-python to 3.12.0 (#26033) (cherry picked from commit cd7738b4b62c84fd60b2228bdda4956db29f244f) --- docker/pulsar/Dockerfile | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index fc38c14bbbe1c..67e2062a72147 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -112,7 +112,7 @@ ARG PULSAR_CLIENT_PYTHON_VERSION RUN pip3 install --break-system-packages --no-cache-dir \ --only-binary \ grpcio==1.78.0 \ - protobuf==6.33.5 \ + protobuf==6.33.6 \ pulsar-client[all]==${PULSAR_CLIENT_PYTHON_VERSION} \ kazoo diff --git a/pom.xml b/pom.xml index 84e83368d4937..07742afb7f193 100644 --- a/pom.xml +++ b/pom.xml @@ -82,7 +82,7 @@ flexible messaging model and an intuitive client API. ${maven.compiler.target} 8 - 3.10.0 + 3.12.0 21 From b70749c8d564748416538ae4b2f6ee501a3e7e3c Mon Sep 17 00:00:00 2001 From: SongOf <46475785+SongOf@users.noreply.github.com> Date: Thu, 18 Jun 2026 06:41:18 +0800 Subject: [PATCH 060/213] [fix][fn] Make exclusiveLeaderProducer volatile in FunctionMetaDataManager (#26046) Co-authored-by: maxlisongsong (cherry picked from commit b8b1e2c57bd7145fc77ac3c5232957490aee9b77) --- .../apache/pulsar/functions/worker/FunctionMetaDataManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java index 944b4e8e34491..92680c60c3e79 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionMetaDataManager.java @@ -72,7 +72,7 @@ public class FunctionMetaDataManager implements AutoCloseable { // The producer of the metadata topic when we are the leader. // Note that this variable serves a double duty. A non-null value // implies we are the leader, while a null value means we are not the leader - private Producer exclusiveLeaderProducer; + private volatile Producer exclusiveLeaderProducer; @Getter private volatile MessageId lastMessageSeen = MessageId.earliest; From 486ab3865e330caaae73b250e97537c82a827b9c Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 18 Jun 2026 06:57:06 +0800 Subject: [PATCH 061/213] [fix][broker] Run topic policy notifications on the topic-ordered executor (#26042) (cherry picked from commit a9278c2c4c26deb0d2a1f6bb02bd0001e65c13a4) Signed-off-by: Zixuan Liu --- .../pulsar/broker/service/AbstractTopic.java | 11 +++ .../pulsar/broker/service/BrokerService.java | 45 ++++++++- .../SystemTopicBasedTopicPoliciesService.java | 93 ++++++++++++------- .../nonpersistent/NonPersistentTopic.java | 4 +- .../service/persistent/PersistentTopic.java | 7 +- ...temTopicBasedTopicPoliciesServiceTest.java | 30 ++++++ 6 files changed, 148 insertions(+), 42 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 988ebe50003b7..60007b0e0d05b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -43,6 +43,7 @@ import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.AtomicLongFieldUpdater; @@ -192,10 +193,16 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener { protected final Clock clock; protected Set additionalSystemCursorNames = new TreeSet<>(); + private final ExecutorService topicPoliciesNotifyThread; public AbstractTopic(String topic, BrokerService brokerService) { this.topic = topic; this.namespace = TopicName.get(topic).getNamespaceObject(); + // Pin the per-topic policies-notify thread once. BrokerService#getTopicPoliciesNotifyThread centralizes + // the topic-to-thread mapping so it stays consistent with SystemTopicBasedTopicPoliciesService. In unit + // tests that construct topics with a mock BrokerService this returns null (the thread is unused there). + this.topicPoliciesNotifyThread = + brokerService.getTopicPoliciesNotifyThread(TopicName.getPartitionedTopicName(topic)); this.clock = brokerService.getClock(); this.brokerService = brokerService; this.producers = new ConcurrentHashMap<>(); @@ -1506,4 +1513,8 @@ public static String getReplicatorDispatchRateKey(String localCluster, String re } return localCluster; } + + protected ExecutorService getPoliciesNotifyThread() { + return topicPoliciesNotifyThread; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 947902644c35f..ebabc48648b56 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -67,6 +67,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.Semaphore; @@ -2736,9 +2737,10 @@ private void handleLocalPoliciesUpdates(NamespaceName namespace) { log.debug("Notifying topic that local policies have changed: {}", name); } topic.ifPresent(t -> { - if (t instanceof PersistentTopic) { - PersistentTopic topic1 = (PersistentTopic) t; - topic1.onLocalPoliciesUpdate(); + if (t instanceof PersistentTopic persistentTopic) { + runOnTopicPoliciesNotifyThread(t, () -> { + persistentTopic.onLocalPoliciesUpdate(); + }); } }); }); @@ -2758,7 +2760,8 @@ private void handlePoliciesUpdates(NamespaceName namespace) { log.info("[{}] updating with {}", namespace, policies); topics.forEach((name, topicFuture) -> { - if (namespace.includes(TopicName.get(name))) { + TopicName topicName = TopicName.get(name); + if (namespace.includes(topicName)) { // If the topic is already created, immediately apply the updated policies, otherwise // once the topic is created it'll apply the policies update topicFuture.thenAccept(topic -> { @@ -2766,7 +2769,11 @@ private void handlePoliciesUpdates(NamespaceName namespace) { log.debug("Notifying topic that policies have changed: {}", name); } - topic.ifPresent(t -> t.onPoliciesUpdate(policies)); + topic.ifPresent(t -> { + runOnTopicPoliciesNotifyThread(t, () -> { + t.onPoliciesUpdate(policies); + }); + }); }); } }); @@ -2777,6 +2784,16 @@ private void handlePoliciesUpdates(NamespaceName namespace) { }, pulsar.getExecutor()); } + private void runOnTopicPoliciesNotifyThread(Topic t, Runnable runnable) { + ExecutorService policiesNotifyThread; + if (t instanceof AbstractTopic abstractTopic) { + policiesNotifyThread = abstractTopic.getPoliciesNotifyThread(); + } else { + policiesNotifyThread = getTopicPoliciesNotifyThread(TopicName.getPartitionedTopicName(t.getName())); + } + policiesNotifyThread.execute(runnable); + } + private void handleDynamicConfigurationUpdates() { DynamicConfigurationResources dynamicConfigResources = null; try { @@ -3624,6 +3641,24 @@ public OrderedExecutor getTopicOrderedExecutor() { return topicOrderedExecutor; } + /** + * Returns the single executor thread used to apply topic-policy updates for the given topic. All + * topic-policy notifications and policy application for a topic must run on this deterministically-chosen + * thread so that they are serialized and never run concurrently. Centralizing the topic-to-thread mapping + * here keeps it consistent between {@link AbstractTopic} and + * {@link SystemTopicBasedTopicPoliciesService} so the two cannot accidentally diverge. + */ + public ExecutorService getTopicPoliciesNotifyThread(TopicName topicName) { + TopicName baseTopicName; + if (topicName.isPartitioned()) { + // for partitioned topics, we need to use the base topic name + baseTopicName = TopicName.get(topicName.getPartitionedTopicName()); + } else { + baseTopicName = topicName; + } + return topicOrderedExecutor.chooseThread(baseTopicName); + } + /** * If per-broker unacked message reached to limit then it blocks dispatcher if its unacked message limit has been * reached to {@link #maxUnackedMsgsPerDispatcher}. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index 19aa0b4e8718d..6c847e0867d74 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -35,6 +35,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -492,16 +493,7 @@ private void notifyListener(Message msg) { if (msg.getValue() == null) { TopicName topicName = TopicName.get(TopicPoliciesService.unwrapEventKey(msg.getKey()) .getPartitionedTopicName()); - List listeners = this.listeners.get(topicName); - if (listeners != null) { - for (TopicPolicyListener listener : listeners) { - try { - listener.onUpdate(null); - } catch (Throwable error) { - log.error("[{}] call listener error.", topicName, error); - } - } - } + notifyListenersForTopic(topicName, null); return; } @@ -511,19 +503,62 @@ private void notifyListener(Message msg) { TopicPoliciesEvent event = msg.getValue().getTopicPoliciesEvent(); TopicName topicName = TopicName.get(event.getDomain(), event.getTenant(), event.getNamespace(), event.getTopic()); - List listeners = this.listeners.get(topicName); - if (listeners != null) { - TopicPolicies policies = event.getPolicies(); - for (TopicPolicyListener listener : listeners) { - try { - listener.onUpdate(policies); - } catch (Throwable error) { - log.error("[{}] call listener error.", topicName, error); - } + notifyListenersForTopic(topicName, event.getPolicies()); + } + + /** + * Notifies the topic-policy listeners registered for {@code topicName} of a policy update. + * + *

The {@link TopicPolicyListener#onUpdate} calls are dispatched to the per-topic ordered executor + * rather than run inline. The reader callbacks that drive notifications (the {@link #initPolicesCache} + * replay loop and {@link #readMorePoliciesAsync}) run on the single, process-wide shared + * {@code broker-client-shared-internal-executor} thread. A listener (e.g. + * {@code PersistentTopic.onUpdate} -> {@code applyUpdatedTopicPolicies}) can perform non-trivial and + * even blocking work, so running it inline serializes and can stall topic-policy loading for every + * namespace (issue #26037). Keying {@code executeOrdered} by {@code topicName} preserves per-topic + * notification ordering. + */ + private void notifyListenersForTopic(TopicName topicName, @Nullable TopicPolicies policies) { + // The per-topic value is a CopyOnWriteArrayList, so iterating it later on the executor thread stays + // safe even if a listener is registered/unregistered between dispatch and execution. + List topicListeners = listeners.get(topicName); + if (topicListeners == null || topicListeners.isEmpty()) { + return; + } + pulsarService.getBrokerService().getTopicPoliciesNotifyThread(topicName).execute(() -> { + internalNotifyTopicListeners(topicName, policies, topicListeners); + }); + } + + // this method should only be called from the topic ordered executor thread + // use notifyListenersForTopic/notifyListenersForTopicAsync instead + private static void internalNotifyTopicListeners(TopicName topicName, @Nullable TopicPolicies policies, + List topicListeners) { + for (TopicPolicyListener listener : topicListeners) { + try { + listener.onUpdate(policies); + } catch (Throwable error) { + log.error("[{}] Error in notifying listener {} on topic policy update.", + topicName, listener, error); } } } + private CompletableFuture notifyListenersForTopicAsync(TopicName topicName, + @Nullable TopicPolicies policies) { + // The per-topic value is a CopyOnWriteArrayList, so iterating it later on the executor thread stays + // safe even if a listener is registered/unregistered between dispatch and execution. + List topicListeners = listeners.get(topicName); + if (topicListeners == null || topicListeners.isEmpty()) { + return CompletableFuture.completedFuture(null); + } + ExecutorService pinnedTopicOrderedExecutor = + pulsarService.getBrokerService().getTopicPoliciesNotifyThread(topicName); + return CompletableFuture.runAsync(() -> { + internalNotifyTopicListeners(topicName, policies, topicListeners); + }, pinnedTopicOrderedExecutor); + } + @Override public CompletableFuture> getTopicPoliciesAsync(TopicName topicName, GetType type) { requireNonNull(topicName); @@ -834,19 +869,13 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp } // replay policy message - policiesCache.forEach(((topicName, topicPolicies) -> { - if (listeners.get(topicName) != null) { - for (TopicPolicyListener listener : listeners.get(topicName)) { - try { - listener.onUpdate(topicPolicies); - } catch (Throwable error) { - log.error("[{}] call listener error.", topicName, error); - } - } - } - })); - - future.complete(null); + List> notifyFutures = new ArrayList<>(); + for (Map.Entry entry : policiesCache.entrySet()) { + TopicName topicName = entry.getKey(); + TopicPolicies policies = entry.getValue(); + notifyFutures.add(notifyListenersForTopicAsync(topicName, policies)); + } + FutureUtil.completeAfter(future, FutureUtil.waitForAll(notifyFutures)); } }); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index d01375dfc8c30..60cffbe89a33d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -164,7 +164,7 @@ private CompletableFuture updateClusterMigrated() { public CompletableFuture initialize() { return brokerService.pulsar().getPulsarResources().getNamespaceResources() .getPoliciesAsync(TopicName.get(topic).getNamespaceObject()) - .thenCompose(optPolicies -> { + .thenComposeAsync(optPolicies -> { final Policies policies; if (optPolicies.isEmpty()) { log.warn("[{}] Policies not present and isEncryptionRequired will be set to false", topic); @@ -179,7 +179,7 @@ public CompletableFuture initialize() { updatePublishRateLimiter(); updateResourceGroupLimiter(); return updateClusterMigrated(); - }); + }, getPoliciesNotifyThread()); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index e0bdbbc789c15..1ffc63c2e8b13 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -532,7 +532,7 @@ public CompletableFuture initialize() { this.isEncryptionRequired = policies.encryption_required; isAllowAutoUpdateSchema = policies.is_allow_auto_update_schema; - }, getOrderedExecutor()) + }, getPoliciesNotifyThread()) .thenCompose(ignore -> initTopicPolicy()) .thenCompose(ignore -> removeOrphanReplicationCursors()) .exceptionally(ex -> { @@ -4798,6 +4798,7 @@ private void updateSubscriptionsDispatcherRateLimiter() { protected CompletableFuture initTopicPolicy() { final var topicPoliciesService = brokerService.pulsar().getTopicPoliciesService(); final var partitionedTopicName = TopicName.getPartitionedTopicName(topic); + return topicPoliciesService.registerListenerAsync(partitionedTopicName, this).thenCompose(registered -> { if (!registered) { return CompletableFuture.completedFuture(null); @@ -4808,11 +4809,11 @@ protected CompletableFuture initTopicPolicy() { return topicPoliciesService.getTopicPoliciesAsync(partitionedTopicName, TopicPoliciesService.GetType.GLOBAL_ONLY) .thenAcceptAsync(optionalPolicies -> optionalPolicies.ifPresent(this::onUpdate), - brokerService.getTopicOrderedExecutor()) + getPoliciesNotifyThread()) .thenCompose(__ -> topicPoliciesService.getTopicPoliciesAsync(partitionedTopicName, TopicPoliciesService.GetType.LOCAL_ONLY)) .thenAcceptAsync(optionalPolicies -> optionalPolicies.ifPresent(this::onUpdate), - brokerService.getTopicOrderedExecutor()); + getPoliciesNotifyThread()); }); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java index 1b5182a454526..dbce03b9d1bbd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java @@ -134,6 +134,36 @@ public void onUpdate(TopicPolicies data) { Assert.assertFalse(systemTopicBasedTopicPoliciesService.listeners.containsKey(topicName)); } + @Test + public void testListenerNotificationRunsOffSharedReaderThread() throws Exception { + // Regression test for #26037: topic-policy listener callbacks must not run on the single, + // process-wide shared "broker-client-shared-internal-executor" reader thread. A slow or blocking + // onUpdate there serializes — and can stall — topic-policy loading for every namespace. The + // notification must instead be dispatched to the per-topic ordered executor ("broker-topic-workers"). + + // Initialize the policy cache and start the change-events reader for the namespace. + systemTopicBasedTopicPoliciesService.updateTopicPoliciesAsync(TOPIC1, false, false, topicPolicies -> + topicPolicies.setMaxConsumerPerTopic(10)).get(); + Awaitility.await().untilAsserted(() -> Assert.assertTrue(systemTopicBasedTopicPoliciesService + .getPoliciesCacheInit(TOPIC1.getNamespaceObject()).isDone())); + + // Register a listener that records the thread its onUpdate runs on. + CompletableFuture onUpdateThreadName = new CompletableFuture<>(); + TopicPolicyListener listener = data -> onUpdateThreadName.complete(Thread.currentThread().getName()); + systemTopicBasedTopicPoliciesService.registerListenerAsync(TOPIC1, listener).get(); + + // A live policy update flows through readMorePoliciesAsync -> notifyListener -> listener.onUpdate. + systemTopicBasedTopicPoliciesService.updateTopicPoliciesAsync(TOPIC1, false, false, topicPolicies -> + topicPolicies.setMaxConsumerPerTopic(20)).get(); + + String threadName = onUpdateThreadName.get(30, TimeUnit.SECONDS); + assertFalse("listener.onUpdate must not run on the shared broker-client reader thread, but ran on: " + + threadName, + threadName.contains("broker-client-shared-internal-executor")); + assertTrue("listener.onUpdate should run on the per-topic ordered executor, but ran on: " + threadName, + threadName.contains("broker-topic-workers")); + } + @Test public void testGetPolicy() throws Exception { From 5de8b35780eb1f7e8d7b88890116586be4ee4706 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 18 Jun 2026 02:44:55 +0300 Subject: [PATCH 062/213] [fix][test] Fix flaky testPrepareInitPoliciesCacheAsyncThrowExceptionAfterCreateReader (#26049) (cherry picked from commit 6f82106ea6c4d508a883ff27875ef279c7c4101c) --- ...temTopicBasedTopicPoliciesServiceTest.java | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java index dbce03b9d1bbd..04b9bc67e7e0a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java @@ -51,6 +51,7 @@ import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.events.PulsarEvent; import org.apache.pulsar.common.naming.NamespaceName; @@ -559,14 +560,28 @@ public void testPrepareInitPoliciesCacheAsyncThrowExceptionAfterCreateReader() t assertTrue(logFound); }); - - // Since cleanPoliciesCacheInitMap() is executed, should add the failed reader into readerCache again. - // Then in SystemTopicBasedTopicPoliciesService, readerCache has a closed reader, - // and policyCacheInitMap do not contain a future. - // To simulate the situation: when getTopicPolicy() execute, it will do prepareInitPoliciesCacheAsync() and - // use a closed reader to read the __change_event topic. Then throw exception - spyReaderCaches.put(NamespaceName.get(NAMESPACE5), readerCompletableFuture); + // The reader.close() above drives readMorePoliciesAsync() into cleanPoliciesCacheInitMap(), which logs + // "Closing the topic policies reader for" and removes the namespace from policyCacheInitMap and readerCaches. + // Now exercise a follow-up prepareInitPoliciesCacheAsync() that reuses a closed reader and must fail in + // initPolicesCache(). Two timing hazards made this flaky (#25081), so both are pinned deterministically: + // 1) A real closed reader's hasMoreEventsAsync() (Reader.hasMessageAvailableAsync()) can answer from cached + // state instead of failing with AlreadyClosedException, in which case initPolicesCache() reaches the end + // of the topic and completes successfully and the expected exception never happens. Use a spy whose + // hasMoreEventsAsync() always fails with AlreadyClosedException. + // 2) A background topic load can re-run prepareInitPoliciesCacheAsync() for the namespace and leave a + // completed init future behind; the next call would then short-circuit through the existing-future + // branch and never re-initialize. Drop any init future right before the call so it re-initializes, and + // stub createSystemTopicClient() so every reader created for this namespace is the failing spy — then + // whichever initialization wins the race fails in the same (asserted) way. + SystemTopicClient.Reader closedReader = Mockito.spy(reader); + Mockito.doReturn(CompletableFuture.failedFuture( + new PulsarClientException.AlreadyClosedException("Reader is already closed"))) + .when(closedReader).hasMoreEventsAsync(); + Mockito.doReturn(CompletableFuture.completedFuture(closedReader)) + .when(spyService).createSystemTopicClient(NamespaceName.get(NAMESPACE5)); + spyReaderCaches.put(NamespaceName.get(NAMESPACE5), CompletableFuture.completedFuture(closedReader)); FieldUtils.writeDeclaredField(spyService, "readerCaches", spyReaderCaches, true); + spyService.policyCacheInitMap.remove(NamespaceName.get(NAMESPACE5)); CompletableFuture prepareFuture = new CompletableFuture<>(); try { From 97b9b9af3fe00f98e304f0c101d9cc25417c2e6a Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 18 Jun 2026 10:29:17 +0300 Subject: [PATCH 063/213] [fix][broker] Prevent topic policy initialization race with a buffering listener wrapper (#26044) (cherry picked from commit b70b3a386c29374cdfa9c5367aacda8a730e4a71) --- .../pulsar/broker/service/AbstractTopic.java | 8 +- .../service/TopicPolicyListenerWrapper.java | 117 +++++++++++++++++ .../nonpersistent/NonPersistentTopic.java | 18 ++- .../service/persistent/PersistentTopic.java | 56 +++++--- .../TopicPolicyListenerWrapperTest.java | 124 ++++++++++++++++++ 5 files changed, 304 insertions(+), 19 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapper.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapperTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 60007b0e0d05b..c1fb51b25fe4f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -636,14 +636,18 @@ protected boolean isProducersExceeded(boolean isRemote) { && maxProducers <= USER_CREATED_PRODUCER_COUNTER_UPDATER.get(this); } + protected TopicPolicyListener getTopicPolicyListener() { + return this; + } + protected void registerTopicPolicyListener() { brokerService.getPulsar().getTopicPoliciesService() - .registerListenerAsync(TopicName.getPartitionedTopicName(topic), this); + .registerListenerAsync(TopicName.getPartitionedTopicName(topic), getTopicPolicyListener()); } protected void unregisterTopicPolicyListener() { brokerService.getPulsar().getTopicPoliciesService() - .unregisterListener(TopicName.getPartitionedTopicName(topic), this); + .unregisterListener(TopicName.getPartitionedTopicName(topic), getTopicPolicyListener()); } protected boolean isSameAddressProducersExceeded(Producer producer) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapper.java new file mode 100644 index 0000000000000..c04d5d30e12d0 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapper.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.broker.service; + +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.common.policies.data.TopicPolicies; + +/** + * This TopicPolicyListener is used as a wrapper for the real TopicPolicyListener. + * This prevents a race condition in initialization where the topic policy state can change while the + * topic policy state is being applied to the topic in PersistentTopic#initTopicPolicy() method or in + * NonPersistentTopic#initialize method. The impact of the race conditions is that the topic policy state would + * be left in an inconsistent state until another update arrives. This is a rare corner case, but possible. + */ +@Slf4j +public class TopicPolicyListenerWrapper implements TopicPolicyListener { + private final TopicPolicyListener realTopicListener; + // The latest value received during initialization, per scope. A null reference means no update was + // received during initialization (the loaded value should be used); an Optional that is present holds the + // received policies, and an empty Optional records that a delete (onUpdate(null)) was received, so the + // loaded value must not be applied. Optional is used because the map-like field cannot itself hold null + // while still distinguishing "not received" (null) from "received a delete" (Optional.empty()). + private Optional latestGlobalPolicies; + private Optional latestLocalPolicies; + private boolean initialized; + private final long createdTimestampNanos = System.nanoTime(); + private static final long INITIALIZATION_WARNING_LOG_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(30); + private int lastIntervalLogged; + + public TopicPolicyListenerWrapper(TopicPolicyListener realTopicListener) { + this.realTopicListener = realTopicListener; + } + + @Override + public synchronized void onUpdate(TopicPolicies data) { + if (initialized) { + realTopicListener.onUpdate(data); + return; + } + + maybeLogWarning(); + + // Record the latest value received during initialization so it can be applied (preferring it over the + // loaded value) in completeInitialization. A received value is stored as Optional.of(data) and a delete + // as Optional.empty(), so the delete is propagated downstream instead of being lost. + if (data == null) { + // A delete (onUpdate(null)) does not carry the global/local scope through the listener interface, + // so record it for both scopes; a later scoped update received during initialization still + // overrides its own scope. + latestGlobalPolicies = Optional.empty(); + latestLocalPolicies = Optional.empty(); + } else if (data.isGlobalPolicies()) { + latestGlobalPolicies = Optional.of(data); + } else { + latestLocalPolicies = Optional.of(data); + } + } + + /** + * Complete initialization of the TopicPolicyListenerWrapper and emit the latest policies to the real listener. + * @param loadedGlobalPolicies the loaded global policies + * @param loadedLocalPolicies the loaded local policies + */ + public synchronized void completeInitialization(TopicPolicies loadedGlobalPolicies, + TopicPolicies loadedLocalPolicies) { + // The listener might have received a newer value (or a delete) than the loaded one while the loading + // was happening; prefer the latest value received during initialization, falling back to the loaded + // value only when nothing was received for that scope. + emitInitialPolicies(latestGlobalPolicies, loadedGlobalPolicies); + emitInitialPolicies(latestLocalPolicies, loadedLocalPolicies); + latestGlobalPolicies = null; + latestLocalPolicies = null; + initialized = true; + } + + private void emitInitialPolicies(Optional latestReceived, TopicPolicies loaded) { + if (latestReceived != null) { + // A value (or a delete) was received during initialization; it supersedes the loaded value. + realTopicListener.onUpdate(latestReceived.orElse(null)); + } else if (loaded != null) { + realTopicListener.onUpdate(loaded); + } + } + + // warn if the initialization takes too long and updates have been received + // this helps detect issues where completeInitialization didn't get called after loading policies + private void maybeLogWarning() { + long durationNanos = System.nanoTime() - createdTimestampNanos; + int warningLogIntervalCount = (int) (durationNanos / INITIALIZATION_WARNING_LOG_INTERVAL_NANOS); + if (warningLogIntervalCount > lastIntervalLogged) { + log.warn("TopicPolicyUpdate buffered. TopicPolicyListenerWrapper initialization phase took too long. " + + "completeInitialization should have been called to complete the phase. " + + "topicPolicyListener={} sinceCreationMs={}", + realTopicListener, TimeUnit.NANOSECONDS.toMillis(durationNanos)); + lastIntervalLogged = warningLogIntervalCount; + } + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 60cffbe89a33d..d220c8a92cefe 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -66,6 +66,7 @@ import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.TopicAttributes; import org.apache.pulsar.broker.service.TopicPolicyListener; +import org.apache.pulsar.broker.service.TopicPolicyListenerWrapper; import org.apache.pulsar.broker.service.TransportCnx; import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; import org.apache.pulsar.broker.service.schema.exceptions.NotExistSchemaException; @@ -127,6 +128,9 @@ protected TopicStats initialValue() { TOPIC_ATTRIBUTES_FIELD_UPDATER = AtomicReferenceFieldUpdater.newUpdater( NonPersistentTopic.class, TopicAttributes.class, "topicAttributes"); + // prevents race conditions in topic policy initialization + private final TopicPolicyListenerWrapper topicPolicyListener = new TopicPolicyListenerWrapper(this); + private static class TopicStats { public double averageMsgSize; public double aggMsgRateIn; @@ -179,7 +183,14 @@ public CompletableFuture initialize() { updatePublishRateLimiter(); updateResourceGroupLimiter(); return updateClusterMigrated(); - }, getPoliciesNotifyThread()); + }, getPoliciesNotifyThread()) + // Complete the topic-policy listener wrapper so buffered and future topic-level policy + // updates are forwarded to this topic. Without this the wrapper stays uninitialized forever + // and all topic-level policy updates are silently dropped. Unlike PersistentTopic, + // non-persistent topics don't load initial topic policies (matching the previous behavior), + // so the loaded values are passed as null. + .thenRunAsync(() -> topicPolicyListener.completeInitialization(null, null), + getPoliciesNotifyThread()); } @Override @@ -1299,4 +1310,9 @@ public TopicAttributes getTopicAttributes() { return TOPIC_ATTRIBUTES_FIELD_UPDATER.updateAndGet(this, old -> old != null ? old : new TopicAttributes(TopicName.get(topic))); } + + @Override + public TopicPolicyListener getTopicPolicyListener() { + return topicPolicyListener; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 1ffc63c2e8b13..493fcc8cfb870 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -61,6 +61,7 @@ import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.function.BiConsumer; import java.util.function.BiFunction; +import java.util.function.Function; import lombok.Getter; import lombok.Value; import org.apache.bookkeeper.client.BKException.BKNoSuchLedgerExistsException; @@ -143,6 +144,8 @@ import org.apache.pulsar.broker.service.TopicEventsListener.EventStage; import org.apache.pulsar.broker.service.TopicEventsListener.TopicEvent; import org.apache.pulsar.broker.service.TopicPoliciesService; +import org.apache.pulsar.broker.service.TopicPolicyListener; +import org.apache.pulsar.broker.service.TopicPolicyListenerWrapper; import org.apache.pulsar.broker.service.TransportCnx; import org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage; import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; @@ -299,6 +302,10 @@ protected TopicStatsHelper initialValue() { // Record the last time max read position is moved forward, unless it's a marker message. @Getter private volatile long lastMaxReadPositionMovedForwardTimestamp = 0; + + // prevents race conditions in topic policy initialization + private final TopicPolicyListenerWrapper topicPolicyListener = new TopicPolicyListenerWrapper(this); + @Getter private final ExecutorService orderedExecutor; @@ -4799,22 +4806,34 @@ protected CompletableFuture initTopicPolicy() { final var topicPoliciesService = brokerService.pulsar().getTopicPoliciesService(); final var partitionedTopicName = TopicName.getPartitionedTopicName(topic); - return topicPoliciesService.registerListenerAsync(partitionedTopicName, this).thenCompose(registered -> { - if (!registered) { - return CompletableFuture.completedFuture(null); - } - if (ExtensibleLoadManagerImpl.isInternalTopic(topic)) { - return CompletableFuture.completedFuture(null); - } - return topicPoliciesService.getTopicPoliciesAsync(partitionedTopicName, - TopicPoliciesService.GetType.GLOBAL_ONLY) - .thenAcceptAsync(optionalPolicies -> optionalPolicies.ifPresent(this::onUpdate), - getPoliciesNotifyThread()) - .thenCompose(__ -> topicPoliciesService.getTopicPoliciesAsync(partitionedTopicName, - TopicPoliciesService.GetType.LOCAL_ONLY)) - .thenAcceptAsync(optionalPolicies -> optionalPolicies.ifPresent(this::onUpdate), - getPoliciesNotifyThread()); - }); + return topicPoliciesService.registerListenerAsync(partitionedTopicName, topicPolicyListener) + .thenCompose(registered -> { + if (!registered) { + return CompletableFuture.completedFuture(null); + } + if (ExtensibleLoadManagerImpl.isInternalTopic(topic)) { + // Internal topics don't load topic-level policies, but the listener wrapper must + // still be initialized so any buffered/future updates are forwarded to the topic + // instead of being silently dropped. + return CompletableFuture.runAsync( + () -> topicPolicyListener.completeInitialization(null, null), + getPoliciesNotifyThread()); + } + // future for fetching global topic policies + CompletableFuture> globalPoliciesFuture = + topicPoliciesService.getTopicPoliciesAsync(partitionedTopicName, + TopicPoliciesService.GetType.GLOBAL_ONLY); + // future for fetching local topic policies + CompletableFuture> localPoliciesFuture = + topicPoliciesService.getTopicPoliciesAsync(partitionedTopicName, + TopicPoliciesService.GetType.LOCAL_ONLY); + return globalPoliciesFuture.thenCombine(localPoliciesFuture, (global, local) -> { + // finally update the topic policies with the latest value or loaded value + return CompletableFuture.runAsync(() -> + topicPolicyListener.completeInitialization(global.orElse(null), local.orElse(null)), + getPoliciesNotifyThread()); + }).thenCompose(Function.identity()); + }); } @VisibleForTesting @@ -4984,4 +5003,9 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { return future; } + + @Override + public TopicPolicyListener getTopicPolicyListener() { + return topicPolicyListener; + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapperTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapperTest.java new file mode 100644 index 0000000000000..0e7553592741d --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapperTest.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import java.util.ArrayList; +import java.util.List; +import org.apache.pulsar.common.policies.data.TopicPolicies; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class TopicPolicyListenerWrapperTest { + + private static TopicPolicies globalPolicies() { + return TopicPolicies.builder().isGlobal(true).build(); + } + + private static TopicPolicies localPolicies() { + return TopicPolicies.builder().isGlobal(false).build(); + } + + private static final class RecordingListener implements TopicPolicyListener { + final List updates = new ArrayList<>(); + + @Override + public void onUpdate(TopicPolicies data) { + updates.add(data); + } + } + + @Test + public void shouldBufferUpdatesUntilInitializedThenForwardLive() { + RecordingListener real = new RecordingListener(); + TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real); + + // Updates received before initialization are buffered, not forwarded. + TopicPolicies bufferedLocal = localPolicies(); + wrapper.onUpdate(bufferedLocal); + assertThat(real.updates).isEmpty(); + + // On completion, the buffered local value wins over the loaded local value; the loaded global value + // is applied since none was buffered. + TopicPolicies loadedGlobal = globalPolicies(); + wrapper.completeInitialization(loadedGlobal, localPolicies()); + assertThat(real.updates).containsExactly(loadedGlobal, bufferedLocal); + + // After initialization, updates are forwarded immediately. + TopicPolicies liveUpdate = localPolicies(); + wrapper.onUpdate(liveUpdate); + assertThat(real.updates).containsExactly(loadedGlobal, bufferedLocal, liveUpdate); + } + + @Test + public void shouldPreferBufferedOverLoadedForBothScopes() { + RecordingListener real = new RecordingListener(); + TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real); + + TopicPolicies bufferedGlobal = globalPolicies(); + TopicPolicies bufferedLocal = localPolicies(); + wrapper.onUpdate(bufferedGlobal); + wrapper.onUpdate(bufferedLocal); + + wrapper.completeInitialization(globalPolicies(), localPolicies()); + assertThat(real.updates).containsExactly(bufferedGlobal, bufferedLocal); + } + + @Test + public void shouldApplyLoadedWhenNothingBuffered() { + RecordingListener real = new RecordingListener(); + TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real); + + TopicPolicies loadedGlobal = globalPolicies(); + TopicPolicies loadedLocal = localPolicies(); + wrapper.completeInitialization(loadedGlobal, loadedLocal); + assertThat(real.updates).containsExactly(loadedGlobal, loadedLocal); + } + + @Test + public void shouldSuppressLoadedValuesWhenDeletedBeforeInitialization() { + RecordingListener real = new RecordingListener(); + TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real); + + // A delete (null) arriving before initialization must not NPE and must not be forwarded yet (#26037). + assertThatCode(() -> wrapper.onUpdate(null)).doesNotThrowAnyException(); + assertThat(real.updates).isEmpty(); + + // The delete supersedes the (now-stale) loaded values: they are not applied, and the delete (null) is + // propagated downstream instead. + wrapper.completeInitialization(globalPolicies(), localPolicies()); + assertThat(real.updates).containsExactly(null, null); + } + + @Test + public void shouldApplyLatestScopedUpdateOverEarlierDeleteDuringInitialization() { + RecordingListener real = new RecordingListener(); + TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real); + + // A delete records empty for both scopes, then a newer global update overrides only the global scope. + wrapper.onUpdate(null); + TopicPolicies newerGlobal = globalPolicies(); + wrapper.onUpdate(newerGlobal); + + wrapper.completeInitialization(globalPolicies(), localPolicies()); + // Global: the newer update wins; Local: the delete (null) wins over the loaded local value. + assertThat(real.updates).containsExactly(newerGlobal, null); + } +} From 204a7e5ad2e84debbf149f3dd40f2ef09e2163c0 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Thu, 18 Jun 2026 17:40:15 +0800 Subject: [PATCH 064/213] [fix][broker] Avoid blocking metadata read on the IO thread when redirecting migrated producers/consumers (#26051) (cherry picked from commit ac053c38fe40723bcee193c1aa6ab5e1d39d1779) Signed-off-by: Zixuan Liu --- .../service/AbstractBaseDispatcher.java | 3 +- .../pulsar/broker/service/AbstractTopic.java | 14 +- .../pulsar/broker/service/Consumer.java | 22 +-- .../pulsar/broker/service/ServerCnx.java | 179 +++++++++++------- .../nonpersistent/NonPersistentTopic.java | 30 +-- .../service/persistent/PersistentTopic.java | 77 ++++---- 6 files changed, 184 insertions(+), 141 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java index 1a1ab2ddca8c8..f95f2a11e05e5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java @@ -496,7 +496,8 @@ protected void checkAndApplyReachedEndOfTopicOrTopicMigration(List con public static void checkAndApplyReachedEndOfTopicOrTopicMigration(PersistentTopic topic, List consumers) { if (topic.isMigrated()) { - consumers.forEach(c -> c.topicMigrated(topic.getMigratedClusterUrl())); + topic.getMigratedClusterUrlAsync() + .thenAccept(clusterUrl -> consumers.forEach(c -> c.topicMigrated(clusterUrl))); } else { consumers.forEach(Consumer::reachedEndOfTopic); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index c1fb51b25fe4f..c48668ddf4ece 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -1455,8 +1455,8 @@ public void updateBrokerSubscribeRate() { subscribeRateInBroker(brokerService.pulsar().getConfiguration())); } - public Optional getMigratedClusterUrl() { - return getMigratedClusterUrl(brokerService.getPulsar(), topic); + public CompletableFuture> getMigratedClusterUrlAsync() { + return getMigratedClusterUrlAsync(brokerService.getPulsar(), topic); } public static CompletableFuture isClusterMigrationEnabled(PulsarService pulsar, @@ -1496,16 +1496,6 @@ private static CompletableFuture isNamespaceMigrationEnabledAsync(Pulsa .thenApply(policies -> policies.isPresent() && policies.get().migrated); } - public static Optional getMigratedClusterUrl(PulsarService pulsar, String topic) { - try { - return getMigratedClusterUrlAsync(pulsar, topic) - .get(pulsar.getPulsarResources().getClusterResources().getOperationTimeoutSec(), TimeUnit.SECONDS); - } catch (Exception e) { - log.warn("[{}] Failed to get migration cluster URL", topic, e); - } - return Optional.empty(); - } - public boolean isSystemCursor(String sub) { return COMPACTION_SUBSCRIPTION.equals(sub) || (additionalSystemCursorNames != null && additionalSystemCursorNames.contains(sub)); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 705b5284ca294..b84de48f25015 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -905,20 +905,16 @@ public void topicMigrated(Optional clusterUrl) { } } - public boolean checkAndApplyTopicMigration() { - if (subscription.isSubscriptionMigrated()) { - Optional clusterUrl = AbstractTopic.getMigratedClusterUrl(cnx.getBrokerService().getPulsar(), - topicName); - if (clusterUrl.isPresent()) { - ClusterUrl url = clusterUrl.get(); - cnx.getCommandSender().sendTopicMigrated(ResourceType.Consumer, consumerId, url.getBrokerServiceUrl(), - url.getBrokerServiceUrlTls()); - // disconnect consumer after sending migrated cluster url - disconnect(); - return true; - } + public CompletableFuture checkAndApplyTopicMigrationAsync() { + if (!subscription.isSubscriptionMigrated()) { + return CompletableFuture.completedFuture(false); } - return false; + return AbstractTopic.getMigratedClusterUrlAsync(cnx.getBrokerService().getPulsar(), topicName) + .thenApply(clusterUrl -> { + // topicMigrated() sends the migrated cluster url and disconnects the consumer if present + topicMigrated(clusterUrl); + return clusterUrl.isPresent(); + }); } /** * Checks if consumer-blocking on unAckedMessages is allowed for below conditions:
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 2bf7d019f3458..1a169d7561bad 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -25,7 +25,7 @@ import static org.apache.pulsar.broker.admin.impl.PersistentTopicsBase.unsafeGetPartitionedTopicMetadataAsync; import static org.apache.pulsar.broker.lookup.TopicLookupBase.lookupTopicAsync; import static org.apache.pulsar.broker.service.ServerCnxThrottleTracker.ThrottleType; -import static org.apache.pulsar.broker.service.persistent.PersistentTopic.getMigratedClusterUrl; +import static org.apache.pulsar.broker.service.persistent.PersistentTopic.getMigratedClusterUrlAsync; import static org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage.ignoreUnrecoverableBKException; import static org.apache.pulsar.common.api.proto.ProtocolVersion.v5; import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH; @@ -1345,8 +1345,9 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { } }); }) - .thenAcceptAsync(consumer -> { - if (consumer.checkAndApplyTopicMigration()) { + .thenComposeAsync(consumer -> consumer.checkAndApplyTopicMigrationAsync() + .thenAcceptAsync(migrated -> { + if (migrated) { log.info("[{}] Disconnecting consumer {} on migrated subscription on topic {} / {}", remoteAddress, consumerId, subscriptionName, topicName); consumers.remove(consumerId, consumerFuture); @@ -1391,7 +1392,7 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { consumers.remove(consumerId, consumerFuture); } - }, ctx.executor()) + }, ctx.executor()), ctx.executor()) .exceptionallyAsync(exception -> { if (exception.getCause() instanceof ConsumerBusyException) { if (log.isDebugEnabled()) { @@ -1402,26 +1403,12 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { exception.getCause().getMessage()); } } else if (exception.getCause() instanceof BrokerServiceException.TopicMigratedException) { - Optional clusterURL = getMigratedClusterUrl(service.getPulsar(), - topicName.toString()); - if (clusterURL.isPresent()) { - log.info("[{}] redirect migrated consumer to topic {}: " - + "consumerId={}, subName={}, {}", remoteAddress, - topicName, consumerId, subscriptionName, exception.getCause().getMessage()); - boolean msgSent = commandSender.sendTopicMigrated(ResourceType.Consumer, consumerId, - clusterURL.get().getBrokerServiceUrl(), - clusterURL.get().getBrokerServiceUrlTls()); - if (!msgSent) { - log.info("consumer client doesn't support topic migration handling {}-{}-{}", - topicName, remoteAddress, consumerId); - } - consumers.remove(consumerId, consumerFuture); - dispatchCloseConsumerEvent(topicName.toString(), consumerId, consumerName, - this.clientSourceAddress(), subscriptionName, subType, - DisconnectInitiator.BROKER); - sendCloseConsumer(consumerId, Optional.empty()); - return null; - } + getMigratedClusterUrlAsync(service.getPulsar(), topicName.toString()) + .exceptionally(e -> Optional.empty()) + .thenAcceptAsync(clusterURL -> redirectOrFailMigratedConsumer(requestId, + consumerId, consumerName, subscriptionName, subType, topicName, + consumerFuture, exception, clusterURL), ctx.executor()); + return null; } else if (exception.getCause() instanceof BrokerServiceException) { log.warn("[{}][{}][{}] Failed to create consumer: consumerId={}, {}", remoteAddress, topicName, subscriptionName, @@ -1459,6 +1446,99 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { }, ctx.executor()); } + private void redirectOrFailMigratedConsumer(long requestId, long consumerId, String consumerName, + String subscriptionName, + SubType subType, TopicName topicName, + CompletableFuture consumerFuture, Throwable exception, + Optional clusterURL) { + if (clusterURL.isPresent()) { + log.info("[{}] redirect migrated consumer to topic {}: consumerId={}, subName={}, {}", remoteAddress, + topicName, consumerId, subscriptionName, exception.getCause().getMessage()); + boolean msgSent = commandSender.sendTopicMigrated(ResourceType.Consumer, consumerId, + clusterURL.get().getBrokerServiceUrl(), + clusterURL.get().getBrokerServiceUrlTls()); + if (!msgSent) { + log.info("consumer client doesn't support topic migration handling {}-{}-{}", + topicName, remoteAddress, consumerId); + } + consumers.remove(consumerId, consumerFuture); + dispatchCloseConsumerEvent(topicName.toString(), consumerId, consumerName, + this.clientSourceAddress(), subscriptionName, subType, + DisconnectInitiator.BROKER); + sendCloseConsumer(consumerId, Optional.empty()); + } else { + // If client timed out, the future would have been completed by subsequent close. + // Send error back to client, only if not completed already. + if (consumerFuture.completeExceptionally(exception)) { + commandSender.sendErrorResponse(requestId, + BrokerServiceException.getClientErrorCode(exception.getCause()), + exception.getCause().getMessage()); + } + consumers.remove(consumerId, consumerFuture); + } + } + + private void redirectOrFailMigratedProducer(long requestId, long producerId, String producerName, + TopicName topicName, CompletableFuture producerFuture, Throwable exception, + Optional clusterURL) { + if (clusterURL.isPresent()) { + log.info("[{}] redirect migrated producer to topic {}: producerId={}, producerName = {}, {}", + remoteAddress, topicName, producerId, producerName, exception.getCause().getMessage()); + boolean msgSent = commandSender.sendTopicMigrated(ResourceType.Producer, producerId, + clusterURL.get().getBrokerServiceUrl(), clusterURL.get().getBrokerServiceUrlTls()); + if (!msgSent) { + log.info("client doesn't support topic migration handling {}-{}-{}", topicName, + remoteAddress, producerId); + } + producers.remove(producerId, producerFuture); + + dispatchCloseProducerEvent(topicName.toString(), producerId, producerName, + this.clientSourceAddress(), DisconnectInitiator.BROKER); + sendCloseProducer(producerId, -1L, Optional.empty()); + } else { + log.error("[{}] Failed to create topic {}, producerId={}", remoteAddress, topicName, producerId, + exception); + if (producerFuture.completeExceptionally(exception)) { + commandSender.sendErrorResponse(requestId, + BrokerServiceException.getClientErrorCode(exception.getCause()), + exception.getCause().getMessage()); + } + producers.remove(producerId, producerFuture); + } + } + + private void redirectOrFailMigratedProducerInQueue(long requestId, long producerId, String producerName, + TopicName topicName, Topic topic, Producer producer, CompletableFuture producerFuture, + Throwable ex, Optional clusterURL) { + if (clusterURL.isPresent() && topic.shouldProducerMigrate()) { + log.info("[{}] redirect migrated producer to topic {}: producerId={}, producerName = {}, {}", + remoteAddress, topicName, producerId, producerName, ex.getCause().getMessage()); + boolean msgSent = commandSender.sendTopicMigrated(ResourceType.Producer, producerId, + clusterURL.get().getBrokerServiceUrl(), clusterURL.get().getBrokerServiceUrlTls()); + if (!msgSent) { + log.info("client doesn't support topic migration handling {}-{}-{}", topic, + remoteAddress, producerId); + } + dispatchCloseProducerEvent(topicName.toString(), producerId, producerName, + this.clientSourceAddress(), DisconnectInitiator.BROKER); + sendCloseProducer(producerId, -1L, Optional.empty()); + } else { + if (clusterURL.isPresent()) { + log.info("Topic {} is migrated but replication backlog exist: " + + "producerId = {}, producerName = {}, {}", topicName, + producerId, producerName, ex.getCause().getMessage()); + } else { + log.warn("[{}] failed producer because migration url not configured topic {}: producerId={}, {}", + remoteAddress, topicName, producerId, ex.getCause().getMessage()); + } + producer.closeNow(true); + if (producerFuture.completeExceptionally(ex)) { + commandSender.sendErrorResponse(requestId, + BrokerServiceException.getClientErrorCode(ex), ex.getMessage()); + } + } + } + private SchemaData getSchema(Schema protocolSchema) { return SchemaData.builder() .data(protocolSchema.getSchemaData()) @@ -1696,23 +1776,11 @@ topicName, TopicOperation.PRODUCE, getAuthenticationData(), getOriginalAuthData( producers.remove(producerId, producerFuture); return null; } else if (cause instanceof BrokerServiceException.TopicMigratedException) { - Optional clusterURL = getMigratedClusterUrl(service.getPulsar(), topicName.toString()); - if (clusterURL.isPresent()) { - log.info("[{}] redirect migrated producer to topic {}: " - + "producerId={}, producerName = {}, {}", remoteAddress, - topicName, producerId, producerName, cause.getMessage()); - boolean msgSent = commandSender.sendTopicMigrated(ResourceType.Producer, producerId, - clusterURL.get().getBrokerServiceUrl(), clusterURL.get().getBrokerServiceUrlTls()); - if (!msgSent) { - log.info("client doesn't support topic migration handling {}-{}-{}", topicName, - remoteAddress, producerId); - } - producers.remove(producerId, producerFuture); - dispatchCloseProducerEvent(topicName.toString(), producerId, producerName, - this.clientSourceAddress(), DisconnectInitiator.BROKER); - sendCloseProducer(producerId, -1L, Optional.empty()); - return null; - } + getMigratedClusterUrlAsync(service.getPulsar(), topicName.toString()) + .exceptionally(e -> Optional.empty()) + .thenAcceptAsync(clusterURL -> redirectOrFailMigratedProducer(requestId, producerId, + producerName, topicName, producerFuture, exception, clusterURL), ctx.executor()); + return null; } // Do not print stack traces for expected exceptions @@ -1807,29 +1875,12 @@ private void buildProducerAndAddTopic(Topic topic, long producerId, String produ producers.remove(producerId, producerFuture); }, ctx.executor()).exceptionallyAsync(ex -> { if (ex.getCause() instanceof BrokerServiceException.TopicMigratedException) { - Optional clusterURL = getMigratedClusterUrl(service.getPulsar(), topic.getName()); - if (clusterURL.isPresent()) { - if (!topic.shouldProducerMigrate()) { - log.info("Topic {} is migrated but replication backlog exist: " - + "producerId = {}, producerName = {}, {}", topicName, - producerId, producerName, ex.getCause().getMessage()); - } else { - log.info("[{}] redirect migrated producer to topic {}: " - + "producerId={}, producerName = {}, {}", remoteAddress, - topicName, producerId, producerName, ex.getCause().getMessage()); - boolean msgSent = commandSender.sendTopicMigrated(ResourceType.Producer, producerId, - clusterURL.get().getBrokerServiceUrl(), clusterURL.get().getBrokerServiceUrlTls()); - if (!msgSent) { - log.info("client doesn't support topic migration handling {}-{}-{}", topic, - remoteAddress, producerId); - } - closeProducer(producer); - return null; - } - } else { - log.warn("[{}] failed producer because migration url not configured topic {}: producerId={}, {}", - remoteAddress, topicName, producerId, ex.getCause().getMessage()); - } + getMigratedClusterUrlAsync(service.getPulsar(), topic.getName()) + .exceptionally(e -> Optional.empty()) + .thenAcceptAsync(clusterURL -> redirectOrFailMigratedProducerInQueue(requestId, producerId, + producerName, topicName, topic, producer, producerFuture, ex, clusterURL), + ctx.executor()); + return null; } else if (ex.getCause() instanceof BrokerServiceException.ProducerFencedException) { if (log.isDebugEnabled()) { log.debug("[{}] Failed to add producer to topic {}: producerId={}, {}", diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index d220c8a92cefe..07a30de8c0dde 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -81,7 +81,6 @@ import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.BacklogQuota; -import org.apache.pulsar.common.policies.data.ClusterPolicies.ClusterUrl; import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.CursorStats; import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; import org.apache.pulsar.common.policies.data.Policies; @@ -338,7 +337,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St false, cnx, cnx.getAuthRole(), metadata, readCompacted, keySharedMeta, MessageId.latest, DEFAULT_CONSUMER_EPOCH, schemaType); if (isMigrated()) { - consumer.topicMigrated(getMigratedClusterUrl()); + getMigratedClusterUrlAsync().thenAccept(consumer::topicMigrated); } addConsumerToSubscription(subscription, consumer).thenRun(() -> { @@ -1017,20 +1016,21 @@ public CompletableFuture checkClusterMigration() { return CompletableFuture.completedFuture(null); } - Optional url = getMigratedClusterUrl(); - if (url.isPresent()) { - this.migrated = true; - producers.forEach((__, producer) -> { - producer.topicMigrated(url); - }); - subscriptions.forEach((__, sub) -> { - sub.getConsumers().forEach((consumer) -> { - consumer.topicMigrated(url); + return getMigratedClusterUrlAsync().thenCompose(url -> { + if (url.isPresent()) { + this.migrated = true; + producers.forEach((__, producer) -> { + producer.topicMigrated(url); }); - }); - return disconnectReplicators().thenCompose(__ -> checkAndUnsubscribeSubscriptions()); - } - return CompletableFuture.completedFuture(null); + subscriptions.forEach((__, sub) -> { + sub.getConsumers().forEach((consumer) -> { + consumer.topicMigrated(url); + }); + }); + return disconnectReplicators().thenCompose(__ -> checkAndUnsubscribeSubscriptions()); + } + return CompletableFuture.completedFuture(null); + }); } private CompletableFuture checkAndUnsubscribeSubscriptions() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 493fcc8cfb870..514cc7daea7c1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -814,18 +814,23 @@ public synchronized void addFailed(ManagedLedgerException exception, Object ctx) // close all producers CompletableFuture disconnectProducersFuture; if (producers.size() > 0) { - List> futures = new ArrayList<>(); // send migration url metadata to producers before disconnecting them - if (isMigrated()) { - if (!shouldProducerMigrate()) { + CompletableFuture sendMigrationUrlFuture; + if (isMigrated() && shouldProducerMigrate()) { + sendMigrationUrlFuture = getMigratedClusterUrlAsync().thenAccept(clusterUrl -> + producers.forEach((__, producer) -> producer.topicMigrated(clusterUrl))); + } else { + if (isMigrated()) { log.info("Topic {} is migrated but replication-backlog exists or " + "subs not created. Closing producers.", topic); - } else { - producers.forEach((__, producer) -> producer.topicMigrated(getMigratedClusterUrl())); } + sendMigrationUrlFuture = CompletableFuture.completedFuture(null); } - producers.forEach((__, producer) -> futures.add(producer.disconnect())); - disconnectProducersFuture = FutureUtil.waitForAll(futures); + disconnectProducersFuture = sendMigrationUrlFuture.thenCompose(v -> { + List> futures = new ArrayList<>(); + producers.forEach((__, producer) -> futures.add(producer.disconnect())); + return FutureUtil.waitForAll(futures); + }); } else { disconnectProducersFuture = CompletableFuture.completedFuture(null); } @@ -3298,39 +3303,39 @@ public CompletableFuture checkClusterMigration() { return CompletableFuture.completedFuture(null); } - Optional clusterUrl = getMigratedClusterUrl(); - - if (!clusterUrl.isPresent()) { - return CompletableFuture.completedFuture(null); - } + return getMigratedClusterUrlAsync().thenCompose(clusterUrl -> { + if (!clusterUrl.isPresent()) { + return CompletableFuture.completedFuture(null); + } - if (isReplicated()) { - if (isReplicationBacklogExist()) { - if (!ledger.isMigrated()) { - log.info("{} applying migration with replication backlog", topic); - ledger.asyncMigrate(); - } - if (log.isDebugEnabled()) { - log.debug("{} has replication backlog and applied migration", topic); + if (isReplicated()) { + if (isReplicationBacklogExist()) { + if (!ledger.isMigrated()) { + log.info("{} applying migration with replication backlog", topic); + ledger.asyncMigrate(); + } + if (log.isDebugEnabled()) { + log.debug("{} has replication backlog and applied migration", topic); + } + return CompletableFuture.completedFuture(null); } - return CompletableFuture.completedFuture(null); } - } - return initMigration().thenCompose(subCreated -> { - migrationSubsCreated = true; - CompletableFuture migrated = !isMigrated() ? ledger.asyncMigrate() - : CompletableFuture.completedFuture(null); - return migrated.thenApply(__ -> { - subscriptions.forEach((name, sub) -> { - if (sub.isSubscriptionMigrated()) { - sub.getConsumers().forEach(Consumer::checkAndApplyTopicMigration); - } - }); - return null; - }).thenCompose(__ -> checkAndDisconnectReplicators()) - .thenCompose(__ -> checkAndUnsubscribeSubscriptions()) - .thenCompose(__ -> checkAndDisconnectProducers()); + return initMigration().thenCompose(subCreated -> { + migrationSubsCreated = true; + CompletableFuture migrated = !isMigrated() ? ledger.asyncMigrate() + : CompletableFuture.completedFuture(null); + return migrated.thenApply(__ -> { + subscriptions.forEach((name, sub) -> { + if (sub.isSubscriptionMigrated()) { + sub.getConsumers().forEach(consumer -> consumer.topicMigrated(clusterUrl)); + } + }); + return null; + }).thenCompose(__ -> checkAndDisconnectReplicators()) + .thenCompose(__ -> checkAndUnsubscribeSubscriptions()) + .thenCompose(__ -> checkAndDisconnectProducers()); + }); }); } From 2e349ed65c3eb743b54f38f799b3deb1cad9de39 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Thu, 18 Jun 2026 04:20:44 -0700 Subject: [PATCH 065/213] [fix][proxy] Avoid blocking the proxy IO thread on a cold broker cache (#26052) (cherry picked from commit 1d7ae01479cc225b9f588c6727d6c5d2e713a0e3) --- .../resources/MetadataStoreCacheLoader.java | 45 ++++++----- .../MetadataStoreCacheLoaderTest.java | 76 +++++++++++++++++++ 2 files changed, 98 insertions(+), 23 deletions(-) create mode 100644 pulsar-broker-common/src/test/java/org/apache/pulsar/broker/resources/MetadataStoreCacheLoaderTest.java diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/MetadataStoreCacheLoader.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/MetadataStoreCacheLoader.java index 29451148da447..7a0a41282824c 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/MetadataStoreCacheLoader.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/MetadataStoreCacheLoader.java @@ -25,7 +25,7 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.commons.collections4.CollectionUtils; import org.apache.pulsar.common.util.FutureUtil; @@ -46,6 +46,7 @@ public class MetadataStoreCacheLoader implements Closeable { private volatile List availableBrokers; private final FutureUtil.Sequencer sequencer; + private final AtomicBoolean refreshInProgress = new AtomicBoolean(false); private final OrderedScheduler orderedExecutor = OrderedScheduler.newSchedulerBuilder().numThreads(8) .name("pulsar-metadata-cache-loader-ordered-cache").build(); @@ -65,40 +66,38 @@ public MetadataStoreCacheLoader(PulsarResources pulsarResources, int operationTi * @throws Exception */ public void init() throws Exception { - Supplier> tryUpdate = () -> { - return loadReportResources.getChildrenAsync(LOADBALANCE_BROKERS_ROOT) - .thenComposeAsync(brokerNodes -> { - return updateBrokerList(brokerNodes).thenRun(() -> { - log.info("Successfully updated broker info {}", brokerNodes); - }); - }) - .exceptionally(ex -> { - log.warn("Error updating broker info after broker list changed", ex); - return null; - }); - }; loadReportResources.getStore().registerListener((n) -> { if (LOADBALANCE_BROKERS_ROOT.equals(n.getPath()) && NotificationType.ChildrenChanged.equals(n.getType())) { - sequencer.sequential(tryUpdate); + sequencer.sequential(this::reloadBrokers); } }); if (loadReportResources.getStore() instanceof MetadataStoreExtended) { ((MetadataStoreExtended) loadReportResources.getStore()).registerSessionListener(sessionEvent -> - sequencer.sequential(tryUpdate)); + sequencer.sequential(this::reloadBrokers)); } // Do initial fetch of brokers list - tryUpdate.get().get(operationTimeoutMs, TimeUnit.MILLISECONDS); + reloadBrokers().get(operationTimeoutMs, TimeUnit.MILLISECONDS); + } + + private CompletableFuture reloadBrokers() { + return loadReportResources.getChildrenAsync(LOADBALANCE_BROKERS_ROOT) + .thenComposeAsync(brokerNodes -> updateBrokerList(brokerNodes).thenRun(() -> + log.info("Successfully updated broker info {}", brokerNodes))) + .exceptionally(ex -> { + log.warn("Error updating broker info after broker list changed", ex); + return null; + }); } public List getAvailableBrokers() { - if (CollectionUtils.isEmpty(availableBrokers)) { - try { - updateBrokerList(loadReportResources.getChildren(LOADBALANCE_BROKERS_ROOT)); - } catch (Exception e) { - log.warn("Error updating broker from zookeeper.", e); - } + List brokers = availableBrokers; + if (CollectionUtils.isEmpty(brokers) && refreshInProgress.compareAndSet(false, true)) { + // Avoid blocking the caller (which may be a Netty IO thread): refresh the cache in the + // background and return the current snapshot. The cache is otherwise kept up to date by the + // metadata-store listener, so an empty snapshot means there are no active brokers. + sequencer.sequential(this::reloadBrokers).whenComplete((__, ex) -> refreshInProgress.set(false)); } - return availableBrokers; + return brokers == null ? new ArrayList<>() : brokers; } @Override diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/resources/MetadataStoreCacheLoaderTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/resources/MetadataStoreCacheLoaderTest.java new file mode 100644 index 0000000000000..e8d4d28153c94 --- /dev/null +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/resources/MetadataStoreCacheLoaderTest.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.resources; + +import static org.apache.pulsar.broker.resources.MetadataStoreCacheLoader.LOADBALANCE_BROKERS_ROOT; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import lombok.Cleanup; +import org.apache.pulsar.metadata.api.MetadataStore; +import org.apache.pulsar.policies.data.loadbalancer.LoadManagerReport; +import org.testng.annotations.Test; + +public class MetadataStoreCacheLoaderTest { + + private MetadataStoreCacheLoader newCacheLoader(LoadManagerReportResources loadReportResources) throws Exception { + MetadataStore store = mock(MetadataStore.class); + PulsarResources pulsarResources = mock(PulsarResources.class); + when(pulsarResources.getLoadReportResources()).thenReturn(loadReportResources); + when(loadReportResources.getStore()).thenReturn(store); + return new MetadataStoreCacheLoader(pulsarResources, 5000); + } + + @Test + public void testGetAvailableBrokersServesCacheWithoutBlocking() throws Exception { + LoadManagerReportResources loadReportResources = mock(LoadManagerReportResources.class); + LoadManagerReport report = mock(LoadManagerReport.class); + when(loadReportResources.getChildrenAsync(LOADBALANCE_BROKERS_ROOT)) + .thenReturn(CompletableFuture.completedFuture(List.of("broker-1"))); + when(loadReportResources.getAsync(LOADBALANCE_BROKERS_ROOT + "/broker-1")) + .thenReturn(CompletableFuture.completedFuture(Optional.of(report))); + + @Cleanup + MetadataStoreCacheLoader loader = newCacheLoader(loadReportResources); + + assertEquals(loader.getAvailableBrokers(), List.of(report)); + // The blocking, synchronous getChildren(...) must never be used: it could stall a Netty IO thread. + verify(loadReportResources, never()).getChildren(anyString()); + } + + @Test + public void testGetAvailableBrokersDoesNotBlockOnEmptyCache() throws Exception { + LoadManagerReportResources loadReportResources = mock(LoadManagerReportResources.class); + when(loadReportResources.getChildrenAsync(LOADBALANCE_BROKERS_ROOT)) + .thenReturn(CompletableFuture.completedFuture(List.of())); + + @Cleanup + MetadataStoreCacheLoader loader = newCacheLoader(loadReportResources); + + assertTrue(loader.getAvailableBrokers().isEmpty()); + verify(loadReportResources, never()).getChildren(anyString()); + } +} From 7d796f42b68a921d1bde30b2749092bdf9461925 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Thu, 18 Jun 2026 04:23:09 -0700 Subject: [PATCH 066/213] [fix][broker] Avoid blocking the dispatcher close path on delayed-delivery tracker close (#26053) (cherry picked from commit 82074aa80c5ca8d849b640a368c254e38e5163e2) --- .../delayed/DelayedDeliveryTracker.java | 10 +++++++ .../bucket/BucketDelayedDeliveryTracker.java | 27 ++++++++++++------- ...PersistentDispatcherMultipleConsumers.java | 8 +++--- ...entDispatcherMultipleConsumersClassic.java | 8 +++--- 4 files changed, 38 insertions(+), 15 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java index 7c954879fe845..96e7151e7ea45 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryTracker.java @@ -86,6 +86,16 @@ public interface DelayedDeliveryTracker extends AutoCloseable { */ void close(); + /** + * Close the subscription tracker and release all resources, completing the returned future once the + * tracker has finished closing. Prefer this over {@link #close()} on asynchronous paths (e.g. inside a + * {@link CompletableFuture} continuation) so the caller is not blocked. + */ + default CompletableFuture closeAsync() { + close(); + return CompletableFuture.completedFuture(null); + } + DelayedDeliveryTracker DISABLE = new DelayedDeliveryTracker() { @Override public boolean addMessage(long ledgerId, long entryId, long deliveryAt) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index 9ed304c73312d..77c4798232b5d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -733,17 +733,26 @@ public synchronized CompletableFuture clear() { } @Override - public synchronized void close() { - super.close(); - lastMutableBucket.close(); - sharedBucketPriorityQueue.close(); - try { - List> completableFutures = immutableBuckets.asMapOfRanges().values().stream() + public void close() { + // Block for AutoCloseable / synchronous callers; asynchronous callers should use closeAsync(). + closeAsync().join(); + } + + @Override + public CompletableFuture closeAsync() { + List> completableFutures; + synchronized (this) { + super.close(); + lastMutableBucket.close(); + sharedBucketPriorityQueue.close(); + completableFutures = immutableBuckets.asMapOfRanges().values().stream() .map(bucket -> bucket.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE)).toList(); - FutureUtil.waitForAll(completableFutures).get(AsyncOperationTimeoutSeconds, TimeUnit.SECONDS); - } catch (Exception e) { - log.warn("[{}] Failed wait to snapshot generate", dispatcher.getName(), e); } + return FutureUtil.waitForAll(completableFutures) + .exceptionally(e -> { + log.warn("[{}] Failed wait to snapshot generate", dispatcher.getName(), e); + return null; + }); } private CompletableFuture cleanImmutableBuckets() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index da6fc52b91638..e51f8a3acf8fc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -621,11 +621,13 @@ public CompletableFuture close(boolean disconnectConsumers, this.delayedDeliveryTracker = Optional.empty(); } - delayedDeliveryTracker.ifPresent(DelayedDeliveryTracker::close); + CompletableFuture closeTrackerFuture = delayedDeliveryTracker + .map(DelayedDeliveryTracker::closeAsync) + .orElseGet(() -> CompletableFuture.completedFuture(null)); dispatchRateLimiter.ifPresent(DispatchRateLimiter::close); - return disconnectConsumers - ? disconnectAllConsumers(false, assignedBrokerLookupData) : CompletableFuture.completedFuture(null); + return closeTrackerFuture.thenCompose(__ -> disconnectConsumers + ? disconnectAllConsumers(false, assignedBrokerLookupData) : CompletableFuture.completedFuture(null)); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java index e87b789d8b9f4..0838b8deab7f1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java @@ -534,11 +534,13 @@ public CompletableFuture close(boolean disconnectConsumers, this.delayedDeliveryTracker = Optional.empty(); } - delayedDeliveryTracker.ifPresent(DelayedDeliveryTracker::close); + CompletableFuture closeTrackerFuture = delayedDeliveryTracker + .map(DelayedDeliveryTracker::closeAsync) + .orElseGet(() -> CompletableFuture.completedFuture(null)); dispatchRateLimiter.ifPresent(DispatchRateLimiter::close); - return disconnectConsumers - ? disconnectAllConsumers(false, assignedBrokerLookupData) : CompletableFuture.completedFuture(null); + return closeTrackerFuture.thenCompose(__ -> disconnectConsumers + ? disconnectAllConsumers(false, assignedBrokerLookupData) : CompletableFuture.completedFuture(null)); } @Override From 8a1b6079d26258397903d110f8977e60b0574155 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Thu, 18 Jun 2026 04:25:00 -0700 Subject: [PATCH 067/213] [fix][broker] Avoid blocking the bundle-throughput lookup on per-bundle metadata reads (#26054) (cherry picked from commit 561f373c1442a81364154aa9880fcb7ada9b89ef) --- .../loadbalance/ModularLoadManager.java | 14 +++ .../impl/ModularLoadManagerImpl.java | 89 ++++++++++--------- .../common/naming/NamespaceBundleFactory.java | 31 ++++--- .../impl/ModularLoadManagerImplTest.java | 2 +- .../namespace/NamespaceServiceTest.java | 2 +- 5 files changed, 84 insertions(+), 54 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManager.java index d608bd6784f29..93059d722e734 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/ModularLoadManager.java @@ -138,8 +138,22 @@ default void writeBrokerDataOnZooKeeper(boolean force) { * * @param bundle * @return bundle data + * @deprecated use {@link #getBundleDataOrDefaultAsync(String)} instead; this method blocks on + * metadata-store reads and must not be called from async or event-loop threads. */ + @Deprecated BundleData getBundleDataOrDefault(String bundle); + /** + * Asynchronously fetch bundle's load report data. + * + * @param bundle + * @return future of the bundle data + */ + @SuppressWarnings("deprecation") + default CompletableFuture getBundleDataOrDefaultAsync(String bundle) { + return CompletableFuture.completedFuture(getBundleDataOrDefault(bundle)); + } + String setNamespaceBundleAffinity(String bundle, String broker); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java index a9d7ddd78e07d..214d12e7ad2c8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java @@ -381,46 +381,55 @@ private boolean checkBundleDataExistInNamespaceBundles(NamespaceBundles namespac // Attempt to local the data for the given bundle in metadata store // If it cannot be found, return the default bundle data. + /** + * @deprecated use {@link #getBundleDataOrDefaultAsync(String)} instead. + */ + @Deprecated @Override public BundleData getBundleDataOrDefault(final String bundle) { - BundleData bundleData = null; - try { - Optional optBundleData = - pulsarResources.getLoadBalanceResources().getBundleDataResources().getBundleData(bundle).join(); - if (optBundleData.isPresent()) { - return optBundleData.get(); - } + return getBundleDataOrDefaultAsync(bundle).join(); + } - Optional optQuota = pulsarResources.getLoadBalanceResources().getQuotaResources() - .getQuota(bundle).join(); - if (optQuota.isPresent()) { - ResourceQuota quota = optQuota.get(); - bundleData = new BundleData(NUM_SHORT_SAMPLES, NUM_LONG_SAMPLES); - // Initialize from existing resource quotas if new API ZNodes do not exist. - final TimeAverageMessageData shortTermData = bundleData.getShortTermData(); - final TimeAverageMessageData longTermData = bundleData.getLongTermData(); - - shortTermData.setMsgRateIn(quota.getMsgRateIn()); - shortTermData.setMsgRateOut(quota.getMsgRateOut()); - shortTermData.setMsgThroughputIn(quota.getBandwidthIn()); - shortTermData.setMsgThroughputOut(quota.getBandwidthOut()); - - longTermData.setMsgRateIn(quota.getMsgRateIn()); - longTermData.setMsgRateOut(quota.getMsgRateOut()); - longTermData.setMsgThroughputIn(quota.getBandwidthIn()); - longTermData.setMsgThroughputOut(quota.getBandwidthOut()); - - // Assume ample history. - shortTermData.setNumSamples(NUM_SHORT_SAMPLES); - longTermData.setNumSamples(NUM_LONG_SAMPLES); - } - } catch (Exception e) { - log.warn("Error when trying to find bundle {} on metadata store: {}", bundle, e); - } - if (bundleData == null) { - bundleData = new BundleData(NUM_SHORT_SAMPLES, NUM_LONG_SAMPLES, defaultStats); - } - return bundleData; + @Override + public CompletableFuture getBundleDataOrDefaultAsync(final String bundle) { + return pulsarResources.getLoadBalanceResources().getBundleDataResources().getBundleData(bundle) + .thenCompose(optBundleData -> { + if (optBundleData.isPresent()) { + return CompletableFuture.completedFuture(optBundleData.get()); + } + return pulsarResources.getLoadBalanceResources().getQuotaResources().getQuota(bundle) + .thenApply(optQuota -> { + if (optQuota.isEmpty()) { + return null; + } + ResourceQuota quota = optQuota.get(); + BundleData bundleData = new BundleData(NUM_SHORT_SAMPLES, NUM_LONG_SAMPLES); + // Initialize from existing resource quotas if new API ZNodes do not exist. + final TimeAverageMessageData shortTermData = bundleData.getShortTermData(); + final TimeAverageMessageData longTermData = bundleData.getLongTermData(); + + shortTermData.setMsgRateIn(quota.getMsgRateIn()); + shortTermData.setMsgRateOut(quota.getMsgRateOut()); + shortTermData.setMsgThroughputIn(quota.getBandwidthIn()); + shortTermData.setMsgThroughputOut(quota.getBandwidthOut()); + + longTermData.setMsgRateIn(quota.getMsgRateIn()); + longTermData.setMsgRateOut(quota.getMsgRateOut()); + longTermData.setMsgThroughputIn(quota.getBandwidthIn()); + longTermData.setMsgThroughputOut(quota.getBandwidthOut()); + + // Assume ample history. + shortTermData.setNumSamples(NUM_SHORT_SAMPLES); + longTermData.setNumSamples(NUM_LONG_SAMPLES); + return bundleData; + }); + }) + .exceptionally(e -> { + log.warn("Error when trying to find bundle {} on metadata store: {}", bundle, e); + return null; + }) + .thenApply(bundleData -> bundleData != null ? bundleData + : new BundleData(NUM_SHORT_SAMPLES, NUM_LONG_SAMPLES, defaultStats)); } // Use the Pulsar client to acquire the namespace bundle stats. @@ -561,7 +570,7 @@ private void updateBundleData() { } else { // Otherwise, attempt to find the bundle data on metadata store. // If it cannot be found, use the latest stats as the first sample. - BundleData currentBundleData = getBundleDataOrDefault(bundle); + BundleData currentBundleData = getBundleDataOrDefaultAsync(bundle).join(); currentBundleData.update(stats); bundleData.put(bundle, currentBundleData); } @@ -879,7 +888,7 @@ public Optional selectBrokerForAssignment(final ServiceUnitId serviceUni private void preallocateBundle(String bundle, String broker) { final BundleData data = loadData.getBundleData().computeIfAbsent(bundle, - key -> getBundleDataOrDefault(bundle)); + key -> getBundleDataOrDefaultAsync(bundle).join()); loadData.getBrokerData().get(broker).getPreallocatedBundleData().put(bundle, data); preallocatedBundleToBroker.put(bundle, broker); @@ -893,7 +902,7 @@ Optional selectBroker(final ServiceUnitId serviceUnit) { synchronized (brokerCandidateCache) { final String bundle = serviceUnit.toString(); final BundleData data = loadData.getBundleData().computeIfAbsent(bundle, - key -> getBundleDataOrDefault(bundle)); + key -> getBundleDataOrDefaultAsync(bundle).join()); brokerCandidateCache.clear(); LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, getAvailableBrokers(), diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleFactory.java index 69f5208ce6711..846f64c32c097 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleFactory.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/common/naming/NamespaceBundleFactory.java @@ -56,6 +56,7 @@ import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.stats.CacheMetricsCollector; import org.apache.pulsar.common.util.Backoff; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.Notification; import org.apache.pulsar.policies.data.loadbalancer.BundleData; import org.slf4j.Logger; @@ -244,19 +245,25 @@ public NamespaceBundle getBundle(TopicName topic) { public CompletableFuture getBundleWithHighestThroughputAsync(NamespaceName nsName) { LoadManager loadManager = pulsar.getLoadManager().get(); if (loadManager instanceof ModularLoadManagerWrapper) { - return getBundlesAsync(nsName).thenApply(bundles -> { - double maxMsgThroughput = -1; - NamespaceBundle bundleWithHighestThroughput = null; - for (NamespaceBundle bundle : bundles.getBundles()) { - BundleData bundleData = ((ModularLoadManagerWrapper) loadManager).getLoadManager() - .getBundleDataOrDefault(bundle.toString()); - if (bundleData.getTopics() > 0 - && bundleData.getLongTermData().totalMsgThroughput() > maxMsgThroughput) { - maxMsgThroughput = bundleData.getLongTermData().totalMsgThroughput(); - bundleWithHighestThroughput = bundle; + return getBundlesAsync(nsName).thenCompose(bundles -> { + List bundleList = bundles.getBundles(); + List> bundleDataFutures = bundleList.stream() + .map(bundle -> ((ModularLoadManagerWrapper) loadManager).getLoadManager() + .getBundleDataOrDefaultAsync(bundle.toString())) + .toList(); + return FutureUtil.waitForAll(bundleDataFutures).thenApply(__ -> { + double maxMsgThroughput = -1; + NamespaceBundle bundleWithHighestThroughput = null; + for (int i = 0; i < bundleList.size(); i++) { + BundleData bundleData = bundleDataFutures.get(i).join(); + if (bundleData.getTopics() > 0 + && bundleData.getLongTermData().totalMsgThroughput() > maxMsgThroughput) { + maxMsgThroughput = bundleData.getLongTermData().totalMsgThroughput(); + bundleWithHighestThroughput = bundleList.get(i); + } } - } - return bundleWithHighestThroughput; + return bundleWithHighestThroughput; + }); }); } return getBundleWithHighestTopicsAsync(nsName); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java index f6061204b62ab..60486c96a1845 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java @@ -1026,7 +1026,7 @@ public void testBundleDataDefaultValue(boolean isV1) throws Exception { // get the bundleData of the first bundle range. // The default value of the bundleData be the same as resourceQuota because the resourceQuota is present. - BundleData defaultBundleData = lm.getBundleDataOrDefault(namespaceBundle.toString()); + BundleData defaultBundleData = lm.getBundleDataOrDefaultAsync(namespaceBundle.toString()).join(); TimeAverageMessageData shortTermData = defaultBundleData.getShortTermData(); TimeAverageMessageData longTermData = defaultBundleData.getLongTermData(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java index 3483ce3809967..32259285b1153 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java @@ -690,7 +690,7 @@ public void testSplitBundleWithHighestThroughput() throws Exception { LoadManager loadManager = pulsar.getLoadManager().get(); Awaitility.await().untilAsserted(() -> { BundleData targetBundleData = ((ModularLoadManagerWrapper) loadManager).getLoadManager() - .getBundleDataOrDefault(namespace + "/" + bundle); + .getBundleDataOrDefaultAsync(namespace + "/" + bundle).join(); assertEquals(targetBundleData.getTopics(), 10); }); From 26d95f221ee303c383a629900c14defb1c521e9e Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Sat, 20 Jun 2026 13:51:53 +0300 Subject: [PATCH 068/213] [fix][sec] Upgrade jline to 4.2.1 and picocli to 4.7.7, drop unused jline2 (#26068) (cherry picked from commit c536dcbd4ce598123dbbb967d1d37788c9a0b8a8) --- distribution/server/pom.xml | 6 ------ distribution/server/src/assemble/LICENSE.bin.txt | 7 +++---- distribution/shell/src/assemble/LICENSE.bin.txt | 6 +++--- pom.xml | 5 ++--- 4 files changed, 8 insertions(+), 16 deletions(-) diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index bfa929124e85a..02911df137cd5 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -93,12 +93,6 @@ ${project.version} - - jline - jline - ${jline.version} - - ${project.groupId} pulsar-package-bookkeeper-storage diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index b2c117519b109..611f3f35b4bd8 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -245,8 +245,8 @@ This projects includes binary packages with the following licenses: The Apache Software License, Version 2.0 * JCommander -- com.beust-jcommander-1.82.jar * Picocli - - info.picocli-picocli-4.7.5.jar - - info.picocli-picocli-shell-jline3-4.7.5.jar + - info.picocli-picocli-4.7.7.jar + - info.picocli-picocli-shell-jline3-4.7.7.jar * High Performance Primitive Collections for Java -- com.carrotsearch-hppc-0.9.1.jar * Jackson - com.fasterxml.jackson.core-jackson-annotations-2.18.6.jar @@ -565,8 +565,7 @@ BSD 3-clause "New" or "Revised" License - com.google.auth-google-auth-library-oauth2-http-1.24.1.jar -- ../licenses/LICENSE-google-auth-library.txt * LevelDB -- (included in org.rocksdb.*.jar) -- ../licenses/LICENSE-LevelDB.txt * JSR305 -- com.google.code.findbugs-jsr305-3.0.2.jar -- ../licenses/LICENSE-JSR305.txt - * JLine -- jline-jline-2.14.6.jar -- ../licenses/LICENSE-JLine.txt - * JLine3 -- org.jline-jline-3.21.0.jar -- ../licenses/LICENSE-JLine.txt + * JLine3 -- org.jline-jline-4.2.1.jar -- ../licenses/LICENSE-JLine.txt * OW2 ASM - org.ow2.asm-asm-9.10.jar -- ../licenses/LICENSE-ASM.txt - org.ow2.asm-asm-commons-9.10.jar -- ../licenses/LICENSE-ASM.txt diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index a465e9b3b4524..8b610881d1fd8 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -310,8 +310,8 @@ This projects includes binary packages with the following licenses: The Apache Software License, Version 2.0 * Picocli - - picocli-4.7.5.jar - - picocli-shell-jline3-4.7.5.jar + - picocli-4.7.7.jar + - picocli-shell-jline3-4.7.7.jar * Jackson - jackson-annotations-2.18.6.jar - jackson-core-2.18.6.jar @@ -426,8 +426,8 @@ The Apache Software License, Version 2.0 * JSpecify -- jspecify-1.0.0.jar BSD 3-clause "New" or "Revised" License + * JLine3 -- jline-4.2.1.jar -- ../licenses/LICENSE-JLine.txt * JSR305 -- jsr305-3.0.2.jar -- ../licenses/LICENSE-JSR305.txt - * JLine3 -- jline-3.21.0.jar -- ../licenses/LICENSE-JLine.txt MIT License * SLF4J -- ../licenses/LICENSE-SLF4J.txt diff --git a/pom.xml b/pom.xml index 07742afb7f193..129d1efb5893b 100644 --- a/pom.xml +++ b/pom.xml @@ -275,8 +275,7 @@ flexible messaging model and an intuitive client API. 3.1.0 2.9.1 0.9.0 - 2.14.6 - 3.21.0 + 4.2.1 0.9.1 2.1.0 3.27.7 @@ -314,7 +313,7 @@ flexible messaging model and an intuitive client API. 2.21.0 ${opentelemetry.instrumentation.version}-alpha 1.37.0 - 4.7.5 + 4.7.7 1.8 0.3.6 3.3.2 From 44eda75718af59c21b5afc167fba596bf17d03d0 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sat, 20 Jun 2026 14:00:29 -0700 Subject: [PATCH 069/213] [fix] functions: Run worker leader-election off the consumer event-listener thread (#26059) (cherry picked from commit ced1280a108709b9fddbb4f8a161fd288c3e2565) --- .../functions/worker/LeaderService.java | 28 ++++++++++++++++++- .../functions/worker/LeaderServiceTest.java | 8 ++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/LeaderService.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/LeaderService.java index e7816f06aacc8..d15b917463550 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/LeaderService.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/LeaderService.java @@ -18,6 +18,11 @@ */ package org.apache.pulsar.functions.worker; +import com.google.common.annotations.VisibleForTesting; +import io.netty.util.concurrent.DefaultThreadFactory; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.function.Supplier; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.Consumer; @@ -42,6 +47,11 @@ public class LeaderService implements AutoCloseable, ConsumerEventListener { private final WorkerConfig workerConfig; private final PulsarClient pulsarClient; private volatile boolean isLeader = false; + // The consumer event listener callbacks (becameActive/becameInactive) run the blocking + // leader-election routines on this dedicated single-threaded executor so that the Pulsar client's + // shared consumer-listener thread is not blocked. The single thread also preserves event ordering. + private final ExecutorService executor = + Executors.newSingleThreadExecutor(new DefaultThreadFactory("function-worker-leader")); static final String COORDINATION_TOPIC_SUBSCRIPTION = "participants"; @@ -89,6 +99,12 @@ public void start() throws PulsarClientException { @Override public void becameActive(Consumer consumer, int partitionId) { + // Run the (blocking) become-leader routine on a dedicated executor so the consumer + // event-listener thread, which is shared with the Pulsar client, is not blocked. + executor.execute(() -> becameActiveInternal(consumer, partitionId)); + } + + private void becameActiveInternal(Consumer consumer, int partitionId) { synchronized (this) { if (isLeader) { return; @@ -148,7 +164,11 @@ public void becameActive(Consumer consumer, int partitionId) { } @Override - public synchronized void becameInactive(Consumer consumer, int partitionId) { + public void becameInactive(Consumer consumer, int partitionId) { + executor.execute(() -> becameInactiveInternal(consumer, partitionId)); + } + + private synchronized void becameInactiveInternal(Consumer consumer, int partitionId) { if (isLeader) { log.info("Worker {} lost the leadership.", consumerName); isLeader = false; @@ -176,10 +196,16 @@ public boolean isLeader() { return isLeader; } + @VisibleForTesting + void joinPendingEventTasks() throws InterruptedException, ExecutionException { + executor.submit(() -> { }).get(); + } + @Override public void close() throws PulsarClientException { if (consumer != null) { consumer.close(); } + executor.shutdown(); } } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/LeaderServiceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/LeaderServiceTest.java index 5c10a59bd1388..03cbd772ab2e6 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/LeaderServiceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/LeaderServiceTest.java @@ -135,6 +135,7 @@ public void testLeaderService() throws Exception { verify(mockClient, times(1)).newConsumer(); listenerHolder.get().becameActive(mockConsumer, 0); + leaderService.joinPendingEventTasks(); assertTrue(leaderService.isLeader()); verify(functionMetadataManager, times(1)).getIsInitialized(); @@ -150,6 +151,7 @@ public void testLeaderService() throws Exception { verify(schedulerManager, times((1))).initialize(any()); listenerHolder.get().becameInactive(mockConsumer, 0); + leaderService.joinPendingEventTasks(); assertFalse(leaderService.isLeader()); verify(functionAssignmentTailer, times(1)).startFromMessage(messageId); @@ -165,6 +167,7 @@ public void testLeaderServiceNoNewScheduling() throws Exception { verify(mockClient, times(1)).newConsumer(); listenerHolder.get().becameActive(mockConsumer, 0); + leaderService.joinPendingEventTasks(); assertTrue(leaderService.isLeader()); verify(functionMetadataManager, times(1)).acquireExclusiveWrite(any()); @@ -176,6 +179,7 @@ public void testLeaderServiceNoNewScheduling() throws Exception { verify(schedulerManager, times((1))).initialize(any()); listenerHolder.get().becameInactive(mockConsumer, 0); + leaderService.joinPendingEventTasks(); assertFalse(leaderService.isLeader()); verify(functionAssignmentTailer, times(1)).start(); @@ -195,6 +199,7 @@ public void testAcquireScheduleManagerExclusiveProducerNotLeaderAnymore() throws when(schedulerManager.acquireExclusiveWrite(any())).thenThrow(new WorkerUtils.NotLeaderAnymore()); listenerHolder.get().becameActive(mockConsumer, 0); + leaderService.joinPendingEventTasks(); // should have failed to become leader assertFalse(leaderService.isLeader()); @@ -211,6 +216,7 @@ public void testAcquireScheduleManagerExclusiveProducerNotLeaderAnymore() throws verify(schedulerManager, times((0))).initialize(any()); listenerHolder.get().becameInactive(mockConsumer, 0); + leaderService.joinPendingEventTasks(); assertFalse(leaderService.isLeader()); verify(functionAssignmentTailer, times(0)).startFromMessage(messageId); @@ -231,6 +237,7 @@ public void testAcquireFunctionMetadataManagerExclusiveProducerNotLeaderAnymore( when(functionMetadataManager.acquireExclusiveWrite(any())).thenThrow(new WorkerUtils.NotLeaderAnymore()); listenerHolder.get().becameActive(mockConsumer, 0); + leaderService.joinPendingEventTasks(); // should have failed to become leader assertFalse(leaderService.isLeader()); @@ -247,6 +254,7 @@ public void testAcquireFunctionMetadataManagerExclusiveProducerNotLeaderAnymore( verify(schedulerManager, times((0))).initialize(any()); listenerHolder.get().becameInactive(mockConsumer, 0); + leaderService.joinPendingEventTasks(); assertFalse(leaderService.isLeader()); verify(functionAssignmentTailer, times(0)).startFromMessage(messageId); From 98bed5a05b13a3058259086a2589ccc0511254c1 Mon Sep 17 00:00:00 2001 From: void-ptr974 Date: Tue, 23 Jun 2026 04:16:10 +0800 Subject: [PATCH 070/213] [fix][broker] Fix geo-replication stuck after a failed publish to the remote cluster (#26002) (cherry picked from commit 9b9e67a9d710f6e4cc4d0665a6cd8dab24c45973) --- .../persistent/PersistentReplicator.java | 3 ++ .../PersistentReplicatorInflightTaskTest.java | 32 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java index 7a32591cf6865..7a1fc364d040f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java @@ -452,6 +452,9 @@ public void sendComplete(Throwable exception, OpSendMsgStats opSendMsgStats) { // cursor should be rewound since it was incremented when readMoreEntries replicator.beforeTerminateOrCursorRewinding(ReasonOfWaitForCursorRewinding.Failed_Publishing); replicator.doRewindCursor(false); + // The failed send has completed from the producer queue perspective. The cursor rewind + // makes the entry readable again, so this in-flight task must release its permit. + inFlightTask.incCompletedEntries(); } else { if (log.isDebugEnabled()) { log.debug("[{}] Message persisted on remote broker", replicator.replicatorId, exception); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java index c825d49caa62e..0c22b14acf748 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.service.persistent; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.testng.Assert.assertEquals; @@ -43,8 +44,11 @@ import org.apache.pulsar.broker.service.BrokerServiceInternalMethodInvoker; import org.apache.pulsar.broker.service.OneWayReplicatorTestBase; import org.apache.pulsar.broker.service.persistent.PersistentReplicator.InFlightTask; +import org.apache.pulsar.broker.service.persistent.PersistentReplicator.ProducerSendCallback; +import org.apache.pulsar.broker.service.persistent.PersistentReplicator.ReasonOfWaitForCursorRewinding; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; import org.awaitility.Awaitility; import org.mockito.invocation.InvocationOnMock; @@ -167,6 +171,34 @@ public void testReadEntriesFailedCompletesInFlightTaskAfterReplicatorTerminated( } } + @Test + public void testFailedPublishCompletesInFlightTask() throws Exception { + PersistentReplicator replicator = spy(getReplicator(topicName)); + doNothing().when(replicator).beforeTerminateOrCursorRewinding(ReasonOfWaitForCursorRewinding.Failed_Publishing); + doNothing().when(replicator).doRewindCursor(false); + doNothing().when(replicator).readMoreEntries(); + + LinkedList inFlightTasks = replicator.inFlightTasks; + List originalTasks = new ArrayList<>(inFlightTasks); + inFlightTasks.clear(); + + try { + InFlightTask task = new InFlightTask(PositionFactory.create(1, 1), 1, replicator.getReplicatorId()); + task.setEntries(Collections.singletonList(mock(Entry.class))); + inFlightTasks.add(task); + assertEquals(replicator.getPermitsIfNoPendingRead(), 999); + + ProducerSendCallback callback = ProducerSendCallback.create(replicator, mock(Entry.class), null, task); + callback.sendComplete(new PulsarClientException.ProducerBlockedQuotaExceededException("mocked"), null); + + assertTrue(task.isDone()); + assertEquals(replicator.getPermitsIfNoPendingRead(), 1000); + } finally { + inFlightTasks.clear(); + inFlightTasks.addAll(originalTasks); + } + } + @Test public void testCreateOrRecycleInFlightTaskIntoQueue() throws Exception { log.info("Starting testCreateOrRecycleInFlightTaskIntoQueue"); From d8a379368afe9d1c0c7a47b32d1d74a0304c65b7 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Fri, 19 Jun 2026 02:41:53 +0800 Subject: [PATCH 071/213] [improve][test]Add test: test/testTopicPartitionCannotBeCreatedAfterTopicDeleted (#26038) (cherry picked from commit 8eaf5517ff788d0703f75f5deb45ac5b8edb0635) --- .../api/SimpleProducerConsumerTest.java | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java index c553950014f6e..8b56d593b4779 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java @@ -110,6 +110,7 @@ import org.apache.pulsar.client.impl.PartitionedProducerImpl; import org.apache.pulsar.client.impl.ProducerBase; import org.apache.pulsar.client.impl.ProducerImpl; +import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.TopicMessageImpl; import org.apache.pulsar.client.impl.TypedMessageBuilderImpl; import org.apache.pulsar.client.impl.crypto.MessageCryptoBc; @@ -5410,6 +5411,65 @@ public void testBacklogAfterCreatedSubscription(boolean trimLegderBeforeGetStats admin.topics().delete(topic, false); } + @Test + public void testTopicPartitionCannotBeCreatedAfterTopicDeleted() throws Exception { + final String topic = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + admin.topics().createPartitionedTopic(topic, 1); + + // Inject an delay: delay to handle channel inactive event, to let the producer delay to reconnect. + ClientBuilderImpl clientBuilder = (ClientBuilderImpl) PulsarClient.builder() + .serviceUrl(lookupUrl.toString()) + .statsInterval(0, TimeUnit.SECONDS) + .connectionsPerBroker(1); + CountDownLatch countDownLatch = new CountDownLatch(1); + PulsarClientImpl pulsarClient = InjectedClientCnxClientBuilder.create(clientBuilder, (conf, eventLoopGroup) -> { + + return new ClientCnx(InstrumentProvider.NOOP, conf, eventLoopGroup) { + + @Override + public void channelInactive(ChannelHandlerContext ctx) throws Exception { + // Delay receiving the event, let producer will not reconnect immediately. + log.info("channel inactive"); + countDownLatch.await(); + super.channelInactive(ctx); + } + }; + }); + + // Producer connected. + Producer producer = pulsarClient.newProducer(Schema.BYTES) + .topic(topic) + .create(); + PersistentTopic persistentTopic1 = (PersistentTopic) pulsar.getBrokerService() + .getTopic(topic + "-partition-0", false) + .get(5, TimeUnit.SECONDS).get(); + Awaitility.await().untilAsserted(() -> { + Assert.assertEquals(persistentTopic1.getProducers().values().size(), 1); + }); + + // Make a network issue which leads to the connection breaks. + org.apache.pulsar.broker.service.Producer serviceProducer = persistentTopic1.getProducers().values() + .iterator().next(); + ServerCnx servercnx = (ServerCnx) serviceProducer.getCnx(); + servercnx.ctx().close(); + // After the connection is break, and before the producer reconnects, the partitioned topic can be deleted + // without "--force". + admin.topics().deletePartitionedTopic(topic); + Awaitility.await().untilAsserted(() -> { + assertTrue(persistentTopic1.isClosingOrDeleting()); + }); + + // Verify: the partition can not be loaded up once the partitioned topic was deleted. + countDownLatch.countDown(); + Thread.sleep(10_000); + assertFalse(producer.isConnected()); + assertFalse(pulsar.getBrokerService().getTopics().containsKey(topic + "-partition-0")); + + // cleanup. + producer.close(); + pulsarClient.close(); + } + /** * The internal producer of replicator will resend messages after reconnected. This test guarantees that the * internal producer will continuously resent messages even though the client side encounters the following bugs. From a0e1e2814c5c56442edd24efe46016d70b7693cb Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 5 May 2026 14:22:56 -0700 Subject: [PATCH 072/213] [fix][test] Make SameAuthParamsLookupAutoClusterFailoverTest less timing-sensitive (#25675) (cherry picked from commit 17c6b139915b4cc299d29a071839a128479f68c9) --- ...thParamsLookupAutoClusterFailoverTest.java | 89 +++++++++---------- 1 file changed, 40 insertions(+), 49 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java index a2e587fef395f..96f1e6421fa4b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java @@ -63,11 +63,11 @@ public Object[][] enabledTls () { }; } - // Each state-convergence phase below waits up to 2 minutes. With 3 phases plus - // cluster startup/teardown, the 240s overall timeout can be tight on slow CI agents - // (the probe timeout is 3s and recoverThreshold=5, so a single slow probe can - // stretch a phase to ~30s). Allow 8 minutes overall to match the per-phase budget. - @Test(dataProvider = "enabledTls", timeOut = 480 * 1000) + // Each state-convergence phase below waits up to 3 minutes. The probe timeout is 3s + // and recoverThreshold=5, so a transient probe failure during recovery resets the + // counter and a phase can need ~30s of healthy probes to recover. With 3 phases plus + // cluster startup/teardown, allow 12 minutes overall to absorb slow CI agents. + @Test(dataProvider = "enabledTls", timeOut = 720 * 1000) public void testAutoClusterFailover(boolean enabledTls) throws Exception { // Start clusters. setup(); @@ -108,63 +108,30 @@ public void testAutoClusterFailover(boolean enabledTls) throws Exception { producer.send("0"); Assert.assertEquals(failover.getCurrentPulsarServiceIndex(), 0); - CompletableFuture checkStatesFuture1 = new CompletableFuture<>(); - executor.submit(() -> { - boolean res = stateArray[0] == PulsarServiceState.Healthy; - res = res & stateArray[1] == PulsarServiceState.Healthy; - res = res & stateArray[2] == PulsarServiceState.Healthy; - checkStatesFuture1.complete(res); - }); - Assert.assertTrue(checkStatesFuture1.join()); + assertStatesEqual(executor, stateArray, + PulsarServiceState.Healthy, PulsarServiceState.Healthy, PulsarServiceState.Healthy); // Test failover 0 --> 2. pulsar1.close(); - Awaitility.await().atMost(120, TimeUnit.SECONDS).untilAsserted(() -> { - CompletableFuture checkStatesFuture2 = new CompletableFuture<>(); - executor.submit(() -> { - boolean res = stateArray[0] == PulsarServiceState.Failed; - res = res & stateArray[1] == PulsarServiceState.Failed; - res = res & stateArray[2] == PulsarServiceState.Healthy; - checkStatesFuture2.complete(res); - }); - Assert.assertTrue(checkStatesFuture2.join()); - producer.send("0->2"); - Assert.assertEquals(failover.getCurrentPulsarServiceIndex(), 2); - }); + awaitStatesAndIndex(executor, stateArray, failover, 2, + PulsarServiceState.Failed, PulsarServiceState.Failed, PulsarServiceState.Healthy); + producer.send("0->2"); // Test recover 2 --> 1. executor.execute(() -> { urlArray[1] = url2; }); - Awaitility.await().atMost(120, TimeUnit.SECONDS).untilAsserted(() -> { - CompletableFuture checkStatesFuture3 = new CompletableFuture<>(); - executor.submit(() -> { - boolean res = stateArray[0] == PulsarServiceState.Failed; - res = res & stateArray[1] == PulsarServiceState.Healthy; - res = res & stateArray[2] == PulsarServiceState.Healthy; - checkStatesFuture3.complete(res); - }); - Assert.assertTrue(checkStatesFuture3.join()); - producer.send("2->1"); - Assert.assertEquals(failover.getCurrentPulsarServiceIndex(), 1); - }); + awaitStatesAndIndex(executor, stateArray, failover, 1, + PulsarServiceState.Failed, PulsarServiceState.Healthy, PulsarServiceState.Healthy); + producer.send("2->1"); // Test recover 1 --> 0. executor.execute(() -> { urlArray[0] = url2; }); - Awaitility.await().atMost(120, TimeUnit.SECONDS).untilAsserted(() -> { - CompletableFuture checkStatesFuture4 = new CompletableFuture<>(); - executor.submit(() -> { - boolean res = stateArray[0] == PulsarServiceState.Healthy; - res = res & stateArray[1] == PulsarServiceState.Healthy; - res = res & stateArray[2] == PulsarServiceState.Healthy; - checkStatesFuture4.complete(res); - }); - Assert.assertTrue(checkStatesFuture4.join()); - producer.send("1->0"); - Assert.assertEquals(failover.getCurrentPulsarServiceIndex(), 0); - }); + awaitStatesAndIndex(executor, stateArray, failover, 0, + PulsarServiceState.Healthy, PulsarServiceState.Healthy, PulsarServiceState.Healthy); + producer.send("1->0"); // cleanup. producer.close(); @@ -186,6 +153,30 @@ public void testInitializeCanOnlyBeCalledOnce() throws Exception { } } + /** + * Wait for the state machine to converge to the expected per-index states and current index. + * The state read happens on the failover executor to avoid races with the periodic check task, + * and producer/lookup operations are kept out of the polling loop so a slow message send does + * not consume the convergence budget. + */ + private static void awaitStatesAndIndex(EventLoopGroup executor, PulsarServiceState[] stateArray, + SameAuthParamsLookupAutoClusterFailover failover, + int expectedIndex, + PulsarServiceState... expectedStates) { + Awaitility.await().atMost(180, TimeUnit.SECONDS).untilAsserted(() -> { + assertStatesEqual(executor, stateArray, expectedStates); + Assert.assertEquals(failover.getCurrentPulsarServiceIndex(), expectedIndex); + }); + } + + private static void assertStatesEqual(EventLoopGroup executor, PulsarServiceState[] stateArray, + PulsarServiceState... expected) throws Exception { + CompletableFuture snapshot = new CompletableFuture<>(); + executor.submit(() -> snapshot.complete(stateArray.clone())); + PulsarServiceState[] actual = snapshot.get(10, TimeUnit.SECONDS); + Assert.assertEquals(actual, expected); + } + @Override protected void cleanupPulsarResources() { // Nothing to do. From 28ac97e8d28c608a8d2cf69395427a63126f1be5 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Mon, 22 Jun 2026 07:11:03 -0700 Subject: [PATCH 073/213] [fix][client] Run the failover health probe off the Netty event-loop thread (#26064) (cherry picked from commit f4fc00c197ffaf09a42f9cb0ab0d89515a88a61e) --- ...meAuthParamsLookupAutoClusterFailoverTest.java | 8 ++++---- .../SameAuthParamsLookupAutoClusterFailover.java | 15 +++++++++------ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java index 96f1e6421fa4b..eb091ff9f1f4f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/SameAuthParamsLookupAutoClusterFailoverTest.java @@ -21,11 +21,11 @@ import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.CA_CERT_FILE_PATH; import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.getTlsFileForClient; import static org.apache.pulsar.client.impl.SameAuthParamsLookupAutoClusterFailover.PulsarServiceState; -import io.netty.channel.EventLoopGroup; import java.net.ServerSocket; import java.util.HashMap; import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import org.apache.pulsar.broker.service.NetworkErrorTestBase; import org.apache.pulsar.broker.service.OneWayReplicatorTestBase; @@ -98,7 +98,7 @@ public void testAutoClusterFailover(boolean enabledTls) throws Exception { .tlsTrustCertsFilePath(CA_CERT_FILE_PATH); } final PulsarClient client = clientBuilder.build(); - final EventLoopGroup executor = WhiteboxImpl.getInternalState(failover, "executor"); + final ScheduledExecutorService executor = WhiteboxImpl.getInternalState(failover, "executor"); final PulsarServiceState[] stateArray = WhiteboxImpl.getInternalState(failover, "pulsarServiceStateArray"); @@ -159,7 +159,7 @@ public void testInitializeCanOnlyBeCalledOnce() throws Exception { * and producer/lookup operations are kept out of the polling loop so a slow message send does * not consume the convergence budget. */ - private static void awaitStatesAndIndex(EventLoopGroup executor, PulsarServiceState[] stateArray, + private static void awaitStatesAndIndex(ScheduledExecutorService executor, PulsarServiceState[] stateArray, SameAuthParamsLookupAutoClusterFailover failover, int expectedIndex, PulsarServiceState... expectedStates) { @@ -169,7 +169,7 @@ private static void awaitStatesAndIndex(EventLoopGroup executor, PulsarServiceSt }); } - private static void assertStatesEqual(EventLoopGroup executor, PulsarServiceState[] stateArray, + private static void assertStatesEqual(ScheduledExecutorService executor, PulsarServiceState[] stateArray, PulsarServiceState... expected) throws Exception { CompletableFuture snapshot = new CompletableFuture<>(); executor.submit(() -> snapshot.complete(stateArray.clone())); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java index 69c74bd28d92a..b137e37996efa 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/SameAuthParamsLookupAutoClusterFailover.java @@ -19,10 +19,11 @@ package org.apache.pulsar.client.impl; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import io.netty.channel.EventLoopGroup; -import io.netty.util.concurrent.ScheduledFuture; import java.util.Arrays; import java.util.HashSet; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -35,7 +36,6 @@ import org.apache.pulsar.client.util.ExecutorProvider; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.common.util.netty.EventLoopUtil; /** * A service URL provider that probes multiple Pulsar service URLs with the same authentication @@ -50,7 +50,7 @@ public class SameAuthParamsLookupAutoClusterFailover implements ServiceUrlProvider { private PulsarClientImpl pulsarClient; - private EventLoopGroup executor; + private ScheduledExecutorService executor; private volatile boolean closed; private ScheduledFuture scheduledCheckTask; @Getter @@ -79,9 +79,12 @@ public synchronized void initialize(PulsarClient client) { } this.currentPulsarServiceIndex = 0; this.pulsarClient = (PulsarClientImpl) client; - this.executor = EventLoopUtil.newEventLoopGroup(1, false, + this.executor = Executors.newSingleThreadScheduledExecutor( new ExecutorProvider.ExtendedThreadFactory("broker-service-url-check")); - scheduledCheckTask = executor.scheduleAtFixedRate(() -> { + // Use fixed-delay (not fixed-rate) scheduling: a probe can block up to its timeout, and with a + // plain single-threaded scheduled executor fixed-rate runs would otherwise pile up back-to-back + // and monopolize the thread. Fixed-delay leaves a gap after each check completes. + scheduledCheckTask = executor.scheduleWithFixedDelay(() -> { try { if (closed) { return; From e3be3e259648b7c98c4a188cb3bbd0640d7604c8 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Mon, 22 Jun 2026 07:12:54 -0700 Subject: [PATCH 074/213] [fix][meta] Run ledger-underreplication notification callbacks off the metadata-store listener thread (#26065) (cherry picked from commit cc96d1c385b0bc8e930579adb9aea742feedd191) --- .../PulsarLedgerUnderreplicationManager.java | 37 +++++++++++++------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java index 2673328b81139..18871be28c116 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java @@ -30,6 +30,7 @@ import com.google.common.base.Joiner; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.TextFormat; +import io.netty.util.concurrent.DefaultThreadFactory; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Arrays; @@ -45,6 +46,8 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeoutException; import java.util.function.Predicate; import java.util.regex.Matcher; @@ -117,6 +120,11 @@ long getLedgerNodeVersion() { private final List> lostBookieRecoveryDelayCallbacks = new ArrayList<>(); + // Registered callbacks can perform synchronous metadata-store reads, so run them on a dedicated + // single-threaded executor instead of the metadata-store notification thread (and outside the lock). + private final ExecutorService notificationCallbackExecutor = + Executors.newSingleThreadExecutor(new DefaultThreadFactory("pulsar-underreplication-notification")); + private static class PulsarUnderreplicatedLedger extends UnderreplicatedLedger { PulsarUnderreplicatedLedger(long ledgerId) { super(ledgerId); @@ -242,13 +250,15 @@ private void handleNotification(Notification n) { callbackList = new ArrayList<>(lostBookieRecoveryDelayCallbacks); lostBookieRecoveryDelayCallbacks.clear(); } - for (BookkeeperInternalCallbacks.GenericCallback callback : callbackList) { - try { - callback.operationComplete(0, null); - } catch (Exception e) { - log.warn("lostBookieRecoveryDelayCallbacks handle error", e); + notificationCallbackExecutor.execute(() -> { + for (BookkeeperInternalCallbacks.GenericCallback callback : callbackList) { + try { + callback.operationComplete(0, null); + } catch (Exception e) { + log.warn("lostBookieRecoveryDelayCallbacks handle error", e); + } } - } + }); return; } if (replicationDisablePath.equals(n.getPath()) && n.getType() == NotificationType.Deleted) { @@ -259,13 +269,15 @@ private void handleNotification(Notification n) { callbackList = new ArrayList<>(replicationEnabledCallbacks); replicationEnabledCallbacks.clear(); } - for (BookkeeperInternalCallbacks.GenericCallback callback : callbackList) { - try { - callback.operationComplete(0, null); - } catch (Exception e) { - log.warn("replicationEnabledCallbacks handle error", e); + notificationCallbackExecutor.execute(() -> { + for (BookkeeperInternalCallbacks.GenericCallback callback : callbackList) { + try { + callback.operationComplete(0, null); + } catch (Exception e) { + log.warn("replicationEnabledCallbacks handle error", e); + } } - } + }); } } } @@ -682,6 +694,7 @@ public void close() throws ReplicationException.UnavailableException { if (log.isDebugEnabled()) { log.debug("close()"); } + notificationCallbackExecutor.shutdownNow(); try { for (Map.Entry e : heldLocks.entrySet()) { store.delete(e.getValue().getLockPath(), Optional.empty()) From dc9c894e13e0c68dff12cd3f88f4f282f5e434e1 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 23 Jun 2026 01:04:48 +0300 Subject: [PATCH 075/213] [fix][broker] Avoid attaching a consumer to a migrated non-persistent topic on subscribe (#26075) (cherry picked from commit d351b6b223be299448cc6512714bd518d4e5f493) --- .../nonpersistent/NonPersistentTopic.java | 66 ++++++++------ .../nonpersistent/NonPersistentTopicTest.java | 89 +++++++++++++++++++ 2 files changed, 127 insertions(+), 28 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 07a30de8c0dde..cd072acb180aa 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -336,37 +336,47 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, false, cnx, cnx.getAuthRole(), metadata, readCompacted, keySharedMeta, MessageId.latest, DEFAULT_CONSUMER_EPOCH, schemaType); - if (isMigrated()) { - getMigratedClusterUrlAsync().thenAccept(consumer::topicMigrated); - } + consumer.checkAndApplyTopicMigrationAsync().thenCompose(migrated -> { + if (migrated) { + // The topic is migrated: checkAndApplyTopicMigrationAsync() has already sent the + // TopicMigrated redirect and disconnected the consumer (which also released the usage + // count taken by handleConsumerAdded above). Skip addConsumerToSubscription so the consumer + // is never attached to the subscription on the old cluster. Sequencing the migration check + // through thenCompose (instead of the previous fire-and-forget thenAccept that raced with + // addConsumerToSubscription) is what guarantees the consumer is not added once migrated. + log.info("[{}][{}] Skipped subscription on migrated topic; consumer {} was redirected", + topic, subscriptionName, consumerId); + future.complete(consumer); + return CompletableFuture.completedFuture(null); + } + return addConsumerToSubscription(subscription, consumer).thenRun(() -> { + if (!cnx.isActive()) { + try { + consumer.close(); + } catch (BrokerServiceException e) { + if (e instanceof ConsumerBusyException) { + log.warn("[{}][{}] Consumer {} {} already connected", topic, subscriptionName, + consumerId, consumerName); + } else if (e instanceof SubscriptionBusyException) { + log.warn("[{}][{}] {}", topic, subscriptionName, e.getMessage()); + } - addConsumerToSubscription(subscription, consumer).thenRun(() -> { - if (!cnx.isActive()) { - try { - consumer.close(); - } catch (BrokerServiceException e) { - if (e instanceof ConsumerBusyException) { - log.warn("[{}][{}] Consumer {} {} already connected", topic, subscriptionName, consumerId, - consumerName); - } else if (e instanceof SubscriptionBusyException) { - log.warn("[{}][{}] {}", topic, subscriptionName, e.getMessage()); + decrementUsageCount(); + future.completeExceptionally(e); + return; } - - decrementUsageCount(); - future.completeExceptionally(e); - return; - } - if (log.isDebugEnabled()) { - log.debug("[{}] [{}] [{}] Subscribe failed -- count: {}", topic, subscriptionName, - consumer.consumerName(), currentUsageCount()); + if (log.isDebugEnabled()) { + log.debug("[{}] [{}] [{}] Subscribe failed -- count: {}", topic, subscriptionName, + consumer.consumerName(), currentUsageCount()); + } + future.completeExceptionally( + new BrokerServiceException.ConnectionClosedException( + "Connection was closed while the opening the cursor ")); + } else { + log.info("[{}][{}] Created new subscription for {}", topic, subscriptionName, consumerId); + future.complete(consumer); } - future.completeExceptionally( - new BrokerServiceException.ConnectionClosedException( - "Connection was closed while the opening the cursor ")); - } else { - log.info("[{}][{}] Created new subscription for {}", topic, subscriptionName, consumerId); - future.complete(consumer); - } + }); }).exceptionally(e -> { Throwable throwable = e.getCause(); if (throwable instanceof ConsumerBusyException) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopicTest.java index 12b6cd2761a19..d09f3068c2f09 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopicTest.java @@ -23,6 +23,7 @@ import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; import java.lang.reflect.Field; +import java.util.Collections; import java.util.Optional; import java.util.UUID; import java.util.concurrent.CompletableFuture; @@ -30,7 +31,9 @@ import lombok.Cleanup; import org.apache.pulsar.broker.service.AbstractTopic; import org.apache.pulsar.broker.service.BrokerTestBase; +import org.apache.pulsar.broker.service.PulsarCommandSender; import org.apache.pulsar.broker.service.SubscriptionOption; +import org.apache.pulsar.broker.service.TransportCnx; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; @@ -39,7 +42,9 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionMode; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.common.api.proto.CommandSubscribe; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.ClusterPolicies.ClusterUrl; import org.apache.pulsar.common.policies.data.TopicStats; import org.awaitility.Awaitility; import org.mockito.Mockito; @@ -126,6 +131,90 @@ public void testCreateNonExistentPartitions() throws PulsarAdminException { } + /** + * Regression test for the migration-redirect race in {@code NonPersistentTopic.internalSubscribe} + * (introduced by PR #26051, which turned a blocking, ordered migration redirect into a fire-and-forget + * async one). When the topic is migrated, the migration check must complete before + * {@code addConsumerToSubscription}, and the consumer must NOT be attached to the subscription on the + * old cluster. The previous code ran {@code getMigratedClusterUrlAsync().thenAccept(consumer::topicMigrated)} + * concurrently with {@code addConsumerToSubscription}, so the consumer could be added before the redirect + * and disconnect ran. The fix sequences the check through {@link org.apache.pulsar.broker.service.Consumer + * #checkAndApplyTopicMigrationAsync()} and skips the add when migrated. + */ + @Test + public void testSubscribeOnMigratedTopicSkipsAddingConsumer() throws Exception { + final String topicName = "non-persistent://prop/ns-abc/migration-race-" + UUID.randomUUID(); + final String subName = "migration-sub"; + + // Materialize the real topic on the broker and acquire namespace-bundle ownership via a client lookup. + @Cleanup + Producer producer = pulsarClient.newProducer().topic(topicName).create(); + NonPersistentTopic realTopic = + (NonPersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get(); + + // Mark the local cluster as migrated so AbstractTopic.getMigratedClusterUrlAsync() (used by + // Consumer.checkAndApplyTopicMigrationAsync()) resolves to a present migrated-cluster URL. + ClusterUrl migratedUrl = new ClusterUrl("http://migrated:8080", "https://migrated:8443", + "pulsar://migrated:6650", "pulsar+ssl://migrated:6651"); + admin.clusters().updateClusterMigration(conf.getClusterName(), true, migratedUrl); + Awaitility.await().atMost(10, TimeUnit.SECONDS).until(() -> + AbstractTopic.getMigratedClusterUrlAsync(pulsar, topicName).get().isPresent()); + + // Spy the topic and force isMigrated() so the subscription is considered migrated. + NonPersistentTopic spyTopic = Mockito.spy(realTopic); + Mockito.doReturn(true).when(spyTopic).isMigrated(); + + // Pre-install a spy subscription so we can assert addConsumer is never invoked when migrated. + NonPersistentSubscription spySubscription = + Mockito.spy(new NonPersistentSubscription(spyTopic, subName, Collections.emptyMap())); + spyTopic.getSubscriptions().put(subName, spySubscription); + + // An active transport connection that records the migration redirect. + PulsarCommandSender commandSender = Mockito.mock(PulsarCommandSender.class); + TransportCnx cnx = Mockito.mock(TransportCnx.class); + Mockito.doReturn(true).when(cnx).isActive(); + Mockito.doReturn(true).when(cnx).isBatchMessageCompatibleVersion(); + Mockito.doReturn("test-role").when(cnx).getAuthRole(); + Mockito.doReturn(pulsar.getBrokerService()).when(cnx).getBrokerService(); + Mockito.doReturn(commandSender).when(cnx).getCommandSender(); + + SubscriptionOption option = SubscriptionOption.builder() + .cnx(cnx) + .subscriptionName(subName) + .consumerId(1L) + .subType(CommandSubscribe.SubType.Shared) + .priorityLevel(0) + .consumerName("consumer-1") + .isDurable(false) + .startMessageId(null) + .metadata(Collections.emptyMap()) + .readCompacted(false) + .initialPosition(CommandSubscribe.InitialPosition.Latest) + .startMessageRollbackDurationSec(0) + .replicatedSubscriptionStateArg(false) + .keySharedMeta(null) + .subscriptionProperties(Optional.empty()) + .build(); + + long usageBefore = spyTopic.currentUsageCount(); + + org.apache.pulsar.broker.service.Consumer consumer = + spyTopic.subscribe(option).get(10, TimeUnit.SECONDS); + assertNotNull(consumer); + + // The migration redirect must have been sent to the client. + Mockito.verify(commandSender).sendTopicMigrated(Mockito.any(), Mockito.eq(1L), + Mockito.eq(migratedUrl.getBrokerServiceUrl()), Mockito.eq(migratedUrl.getBrokerServiceUrlTls())); + + // The core regression assertion: a migrated topic must never attach the consumer to the + // subscription. The buggy code added it before the async redirect/disconnect could run. + Mockito.verify(spySubscription, Mockito.never()).addConsumer(Mockito.any()); + assertTrue(spySubscription.getConsumers().isEmpty()); + + // No usage-count leak: handleConsumerAdded's increment is balanced by the disconnect's removeConsumer. + assertEquals(spyTopic.currentUsageCount(), usageBefore); + } + @Test public void testSubscriptionsOnNonPersistentTopic() throws Exception { final String topicName = "non-persistent://prop/ns-abc/topic_" + UUID.randomUUID(); From a8c0ba405f294c277e972d4355cddbf74070c1c9 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 23 Jun 2026 16:33:40 +0800 Subject: [PATCH 076/213] [feat][broker] Expose managed ledger properties via topic internal stats (#26079) (cherry picked from commit 6d0d99b7c4241b376e4e532183691f8ee48df9ba) --- .../mledger/impl/ManagedLedgerImpl.java | 4 +- .../service/persistent/PersistentTopic.java | 1 + .../pulsar/broker/admin/AdminApi2Test.java | 38 +++++++++++++++++++ .../data/ManagedLedgerInternalStats.java | 3 ++ 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 31f8aafc18302..52f1201e32cc2 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -5007,7 +5007,9 @@ public CompletableFuture getManagedLedgerInternalSta stats.lastConfirmedEntry = this.getLastConfirmedEntry().toString(); stats.state = this.getState().toString(); - stats.cursors = new HashMap(); + stats.properties = new HashMap<>(propertiesMap); + + stats.cursors = new HashMap<>(); this.getCursors().forEach(c -> { ManagedCursorImpl cursor = (ManagedCursorImpl) c; PersistentTopicInternalStats.CursorStats cs = new PersistentTopicInternalStats.CursorStats(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 514cc7daea7c1..ba1a4c26280e1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -3100,6 +3100,7 @@ public CompletableFuture getInternalStats(boolean stats.lastConfirmedEntry = ledgerInternalStats.getLastConfirmedEntry(); stats.state = ledgerInternalStats.getState(); stats.ledgers = ledgerInternalStats.ledgers; + stats.properties = ledgerInternalStats.getProperties(); // Add ledger info for compacted topic ledger if exist. LedgerInfo info = new LedgerInfo(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java index c3703c5e9b938..fe8926142a6e8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java @@ -69,7 +69,9 @@ import lombok.Cleanup; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.commons.lang3.reflect.FieldUtils; @@ -1230,6 +1232,42 @@ public void testUpdatePropertiesOnNonExistentTopic() throws Exception { } } + @Test + public void testGetInternalStatsWithProperties() throws Exception { + final var namespace = newUniqueName(defaultTenant + "/ns2"); + final var topicName = "persistent://" + namespace + "/testGetInternalStatsWithProperties"; + admin.namespaces().createNamespace(namespace); + + final var topicProperties = Map.of("key1", "value1", "key2", "value2"); + admin.topics().createNonPartitionedTopic(topicName, topicProperties); + + var stats = admin.topics().getInternalStats(topicName); + assertEquals(stats.properties, topicProperties); + + var persistentTopic = (PersistentTopic) pulsar.getBrokerService().getTopicIfExists(topicName).get() + .orElseThrow(); + final var future = new CompletableFuture>(); + persistentTopic.getManagedLedger().asyncSetProperty("new-key", "new-value", + new AsyncCallbacks.UpdatePropertiesCallback() { + @Override + public void updatePropertiesComplete(Map properties, Object ctx) { + future.complete(properties); + } + + @Override + public void updatePropertiesFailed(ManagedLedgerException exception, Object ctx) { + future.completeExceptionally(exception); + } + }, null); + assertEquals(future.get(), Map.of("key1", "value1", "key2", "value2", "new-key", "new-value")); + + admin.namespaces().unload(namespace); + persistentTopic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, true).get() + .orElseThrow(); + stats = admin.topics().getInternalStats(topicName); + assertEquals(stats.properties, Map.of("key1", "value1", "key2", "value2", "new-key", "new-value")); + } + @Test public void testNonPersistentTopics() throws Exception { final String namespace = newUniqueName(defaultTenant + "/ns2"); diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ManagedLedgerInternalStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ManagedLedgerInternalStats.java index b68b6308c8f3b..5e5784e70681d 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ManagedLedgerInternalStats.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ManagedLedgerInternalStats.java @@ -71,6 +71,9 @@ public class ManagedLedgerInternalStats { /** The list of all cursors on this topic. Each subscription in the topic stats has a cursor. */ public Map cursors; + /** The properties map of the managed ledger. */ + public Map properties; + /** * Ledger information. */ From 591151a8a86bf078efb3f25da96e56eefccda192 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Wed, 24 Jun 2026 17:32:21 +0800 Subject: [PATCH 077/213] [fix][broker]Do not trigger topic GC if replication is still active (#25915) (cherry picked from commit 4fd22bbe80b1812b7840cefe577d2377fd1b5472) Signed-off-by: Zixuan Liu --- .../pulsar/broker/ServiceConfiguration.java | 14 ++ .../broker/service/AbstractReplicator.java | 14 +- .../pulsar/broker/service/AbstractTopic.java | 27 +++ .../pulsar/broker/service/BrokerService.java | 12 ++ .../pulsar/broker/service/Replicator.java | 4 - .../NonPersistentReplicator.java | 6 + .../persistent/GeoPersistentReplicator.java | 7 +- .../persistent/PersistentReplicator.java | 81 ++++++-- .../service/persistent/PersistentTopic.java | 96 +-------- .../ReplicatedSubscriptionsController.java | 4 - .../service/persistent/ShadowReplicator.java | 7 +- .../broker/service/OneWayReplicatorTest.java | 194 ++++++++++++++++++ .../service/OneWayReplicatorTestBase.java | 11 +- ...yReplicatorUsingGlobalPartitionedTest.java | 10 + .../OneWayReplicatorUsingGlobalZKTest.java | 78 +++++++ .../broker/service/PersistentTopicTest.java | 25 ++- .../service/ReplicationTopicGcTest.java | 8 + .../pulsar/broker/service/ReplicatorTest.java | 33 ++- .../PersistentReplicatorInflightTaskTest.java | 13 ++ 19 files changed, 489 insertions(+), 155 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 80edc42cef0d6..ce8b52751c0f6 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -758,6 +758,20 @@ The max allowed delay for delayed delivery (in milliseconds). If the broker rece ) private Integer brokerDeleteInactiveTopicsMaxInactiveDurationSeconds = null; + @FieldContext( + category = CATEGORY_POLICIES, + dynamic = true, + doc = "Time in seconds that a persistent geo-replication replicator may stay idle before the broker" + + " disconnects its replication producer. A replicator is eligible only when it has no backlog and" + + " has not read entries for replication processing for longer than this threshold. Disconnecting" + + " only releases the idle producer; the replicator and its cursor remain available, and the" + + " producer is recreated automatically when new messages need to be replicated. Set this value to" + + " 0 or a negative value to disable idle-replicator disconnection. The check runs with the" + + " inactive-topic monitor, whose interval is brokerDeleteInactiveTopicsFrequencySeconds, and only" + + " when brokerDeleteInactiveTopicsEnabled is true. The default is 86400 seconds (24 hours)." + ) + private int brokerReplicationInactiveThresholdSeconds = 24 * 3600; + @FieldContext( category = CATEGORY_POLICIES, dynamic = true, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java index e86c84feb2da3..9c7dc090243d2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java @@ -85,6 +85,11 @@ public abstract class AbstractReplicator implements Replicator { private static final AtomicReferenceFieldUpdater ATTRIBUTES_UPDATER = AtomicReferenceFieldUpdater.newUpdater(AbstractReplicator.class, Attributes.class, "attributes"); + protected volatile long latestPublishTime = System.currentTimeMillis(); + + // The estimated time when the producer connection is successful, "0" means it will be connected immediately. + protected volatile long estimatedTimeStampProducerConnected = 0; + public enum State { /** * This enum has two mean meanings： @@ -166,7 +171,7 @@ protected CompletableFuture prepareCreateProducer() { return CompletableFuture.completedFuture(null); } - public void startProducer() { + protected void startProducer() { // Guarantee only one task call "producerBuilder.createAsync()". Pair setStartingRes = compareSetAndGetState(State.Disconnected, State.Starting); if (!setStartingRes.getLeft()) { @@ -200,12 +205,14 @@ public void startProducer() { builderImpl.getConf().setNonPartitionedTopicExpected(true); builderImpl.getConf().setReplProducer(true); return producerBuilder.createAsync().thenAccept(producer -> { + estimatedTimeStampProducerConnected = 0; setProducerAndTriggerReadEntries(producer); }); }).exceptionally(ex -> { Pair setDisconnectedRes = compareSetAndGetState(State.Starting, State.Disconnected); if (setDisconnectedRes.getLeft()) { long waitTimeMs = backOff.next(); + estimatedTimeStampProducerConnected = System.currentTimeMillis() + waitTimeMs; log.warn("[{}] Failed to create remote producer ({}), retrying in {} s", replicatorId, ex.getMessage(), waitTimeMs / 1000.0); // BackOff before retrying @@ -308,8 +315,7 @@ protected CompletableFuture isLocalTopicActive() { /** * This method only be used by {@link PersistentTopic#checkGC} now. */ - @Override - public CompletableFuture disconnect() { + protected CompletableFuture disconnect() { long backlog = getNumberOfEntriesInBacklog(); if (backlog > 0) { CompletableFuture disconnectFuture = new CompletableFuture<>(); @@ -387,8 +393,6 @@ protected CompletableFuture closeProducerAsync(boolean closeTheStartingPro Pair setDisconnectedRes = compareSetAndGetState(State.Disconnecting, State.Disconnected); if (setDisconnectedRes.getLeft()) { this.producer = null; - // deactivate further read - disableReplicatorRead(); return; } if (setDisconnectedRes.getRight() == State.Terminating diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index c48668ddf4ece..e036f9e1e2e2d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -69,6 +69,7 @@ import org.apache.pulsar.broker.service.BrokerServiceException.TopicMigratedException; import org.apache.pulsar.broker.service.BrokerServiceException.TopicTerminatedException; import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter; +import org.apache.pulsar.broker.service.persistent.PersistentReplicator; import org.apache.pulsar.broker.service.plugin.EntryFilter; import org.apache.pulsar.broker.service.schema.SchemaRegistryService; import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; @@ -740,6 +741,19 @@ protected Consumer getActiveConsumer(Subscription subscription) { return null; } + protected boolean hasProducersActive() { + return !producers.isEmpty(); + } + + protected boolean hasActiveReplicators() { + for (Replicator replicator : getReplicators().values()) { + if (replicator.isConnected()) { + return true; + } + } + return false; + } + protected boolean hasLocalProducers() { if (producers.isEmpty()) { return false; @@ -752,6 +766,19 @@ protected boolean hasLocalProducers() { return false; } + public void disconnectReplicatorsIfNoTrafficAndBacklog() { + for (Replicator replicator : getReplicators().values()) { + if (replicator instanceof PersistentReplicator persistentReplicator) { + persistentReplicator.disconnectIfNoTrafficAndBacklog(); + } + } + for (Replicator replicator : getShadowReplicators().values()) { + if (replicator instanceof PersistentReplicator persistentReplicator) { + persistentReplicator.disconnectIfNoTrafficAndBacklog(); + } + } + } + @Override public String toString() { return MoreObjects.toStringHelper(this).add("topic", topic).toString(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index ebabc48648b56..1f7983306dc52 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -714,6 +714,10 @@ protected void startInactivityMonitor() { int interval = pulsar().getConfiguration().getBrokerDeleteInactiveTopicsFrequencySeconds(); inactivityMonitor.scheduleAtFixedRateNonConcurrently(() -> checkGC(), interval, interval, TimeUnit.SECONDS); + if (pulsar().getConfig().getBrokerReplicationInactiveThresholdSeconds() > 0) { + inactivityMonitor.scheduleAtFixedRateNonConcurrently(() -> checkInactiveReplication(), interval, + interval, TimeUnit.SECONDS); + } } // Deduplication info checker @@ -2399,6 +2403,14 @@ public void checkGC() { forEachTopic(Topic::checkGC); } + public void checkInactiveReplication() { + forEachTopic(topic -> { + if (topic instanceof AbstractTopic abstractTopic) { + abstractTopic.disconnectReplicatorsIfNoTrafficAndBacklog(); + } + }); + } + public void checkClusterMigration() { forEachTopic(Topic::checkClusterMigration); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java index 64d03f6aa95dd..cea4a0a8b6296 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Replicator.java @@ -26,16 +26,12 @@ public interface Replicator { - void startProducer(); - Topic getLocalTopic(); ReplicatorStatsImpl computeStats(); CompletableFuture terminate(); - CompletableFuture disconnect(); - void updateRates(); String getRemoteCluster(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java index 38e1894c17854..0d66bddd378c6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java @@ -68,6 +68,11 @@ protected String getProducerName() { return getReplicatorName(replicatorPrefix, localCluster) + REPL_PRODUCER_NAME_DELIMITER + remoteCluster; } + @Override + public void startProducer() { + super.startProducer(); + } + @Override protected void setProducerAndTriggerReadEntries(Producer producer) { this.producer = (ProducerImpl) producer; @@ -86,6 +91,7 @@ protected void setProducerAndTriggerReadEntries(Producer producer) { } public void sendMessage(Entry entry) { + latestPublishTime = System.currentTimeMillis(); if ((STATE_UPDATER.get(this) == State.Started) && isWritable()) { int length = entry.getLength(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java index 49260f9b6a06a..b4a518c51df90 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java @@ -89,8 +89,7 @@ protected boolean replicateEntries(List entries, final InFlightTask inFli try { // This flag is set to true when we skip at least one local message, // in order to skip remaining local messages. - boolean isLocalMessageSkippedOnce = false; - boolean skipRemainingMessages = inFlightTask.isSkipReadResultDueToCursorRewind(); + boolean skipRemainingMessages = false; for (int i = 0; i < entries.size(); i++) { Entry entry = entries.get(i); // Skip the messages since the replicator need to fetch the schema info to replicate the schema to the @@ -171,14 +170,14 @@ protected boolean replicateEntries(List entries, final InFlightTask inFli continue; } - if (STATE_UPDATER.get(this) != State.Started || isLocalMessageSkippedOnce) { + if (STATE_UPDATER.get(this) != State.Started || inFlightTask.isSkipReadResultDueToCursorRewind()) { // The producer is not ready yet after having stopped/restarted. Drop the message because it will // recover when the producer is ready if (log.isDebugEnabled()) { log.debug("[{}] Dropping read message at {} because producer is not ready", replicatorId, entry.getPosition()); } - isLocalMessageSkippedOnce = true; + skipRemainingMessages = true; inFlightTask.incCompletedEntries(); entry.release(); msg.recycle(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java index 7a1fc364d040f..9fdfd590c4547 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.broker.service.persistent; +import static org.apache.pulsar.broker.service.AbstractReplicator.State.Disconnected; +import static org.apache.pulsar.broker.service.AbstractReplicator.State.Disconnecting; import static org.apache.pulsar.broker.service.AbstractReplicator.State.Started; import static org.apache.pulsar.broker.service.AbstractReplicator.State.Starting; import static org.apache.pulsar.broker.service.AbstractReplicator.State.Terminated; @@ -51,6 +53,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.CursorAlreadyClosedException; import org.apache.bookkeeper.mledger.ManagedLedgerException.TooManyRequestsException; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.PulsarServerException; @@ -151,13 +154,6 @@ public PersistentReplicator(String localCluster, PersistentTopic localTopic, Man @Override protected void setProducerAndTriggerReadEntries(Producer producer) { - // Repeat until there are no read operations in progress - if (STATE_UPDATER.get(this) == State.Starting && hasPendingRead() && !cursor.cancelPendingReadRequest()) { - brokerService.getPulsar().getExecutor() - .schedule(() -> setProducerAndTriggerReadEntries(producer), 10, TimeUnit.MILLISECONDS); - return; - } - /** * 1. Try change state to {@link Started}. * 2. Atoms modify multiple properties if change state success, to avoid another thread get a null value @@ -324,6 +320,30 @@ private AvailablePermits getRateLimiterAvailablePermits(int availablePermits) { return new AvailablePermits((int) availablePermitsOnMsg, availablePermitsOnByte); } + public void disconnectIfNoTrafficAndBacklog() { + // Disabled the feature. + int threshold = brokerService.getPulsar().getConfig().getBrokerReplicationInactiveThresholdSeconds(); + if (threshold <= 0) { + return; + } + // Has backlog. + long backlog = getNumberOfEntriesInBacklog(); + if (backlog > 0) { + return; + } + // Already disconnected. + if (state != Started) { + return; + } + + // Disconnect if no backlog and no traffic for a long time. + if (System.currentTimeMillis() - latestPublishTime > threshold * 1000L) { + log.info("Disconnecting replication producers since no producer is active for a long time." + + " brokerReplicationInactiveThresholdSeconds: {}", threshold); + disconnect(); + } + } + protected void readMoreEntries() { if (state.equals(Terminated) || state.equals(Terminating)) { return; @@ -389,6 +409,48 @@ public void readEntriesComplete(List entries, Object ctx) { log.debug("[{}] Read entries complete of {} messages", replicatorId, entries.size()); } InFlightTask inFlightTask = (InFlightTask) ctx; + + latestPublishTime = System.currentTimeMillis(); + // Release memory if terminated. + if (state == State.Terminated || state == State.Terminating + || inFlightTask.isSkipReadResultDueToCursorRewind()) { + for (Entry entry : entries) { + inFlightTask.incCompletedEntries(); + entry.release(); + } + return; + } + + // Retry to trigger read completes if it is not started. + ManagedLedgerImpl ml = (ManagedLedgerImpl) cursor.getManagedLedger(); + Runnable retryReplicateEntries = () -> { + long estimatedTimeStampProducerConnected = this.estimatedTimeStampProducerConnected; + long delayMillis; + if (estimatedTimeStampProducerConnected > System.currentTimeMillis()) { + delayMillis = (estimatedTimeStampProducerConnected - System.currentTimeMillis()) + 100; + } else { + delayMillis = 100; + } + ml.getScheduledExecutor().schedule(() -> { + ml.getExecutor().execute(() -> { + readEntriesComplete(entries, ctx); + }); + }, delayMillis, TimeUnit.MILLISECONDS); + }; + + // Retry. + if (state == Disconnecting || state == Starting) { + retryReplicateEntries.run(); + return; + } + // Start producer and retry. + if (state == Disconnected) { + startProducer(); + retryReplicateEntries.run(); + return; + } + + // After set entries, the next reading can be start. inFlightTask.setEntries(entries); // After the replicator starts, the speed will be gradually increased. @@ -1017,15 +1079,10 @@ protected CompletableFuture beforeDisconnect() { .TopicBusyException("Cannot close a replicator with backlog")); } } - beforeTerminateOrCursorRewinding(ReasonOfWaitForCursorRewinding.Disconnecting); return CompletableFuture.completedFuture(null); } } - protected void afterDisconnected() { - doRewindCursor(false); - } - protected void beforeTerminateOrCursorRewinding(ReasonOfWaitForCursorRewinding reason) { synchronized (inFlightTasks) { boolean hasCanceledPendingRead = cursor.cancelPendingReadRequest(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index ba1a4c26280e1..388de5ceb8390 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -29,7 +29,6 @@ import com.carrotsearch.hppc.ObjectObjectHashMap; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Sets; import io.netty.buffer.ByteBuf; import io.netty.util.concurrent.FastThreadLocal; import java.io.IOException; @@ -866,9 +865,7 @@ public CompletableFuture> addProducer(Producer producer, CompletableFuture producerQueuedFuture) { return super.addProducer(producer, producerQueuedFuture).thenCompose(topicEpoch -> { messageDeduplication.producerAdded(producer.getProducerName()); - - // Start replication producers if not already - return startReplProducers().thenApply(__ -> topicEpoch); + return CompletableFuture.completedFuture(topicEpoch); }); } @@ -915,46 +912,6 @@ private boolean hasRemoteProducers() { return false; } - public CompletableFuture startReplProducers() { - // read repl-cluster from policies to avoid restart of replicator which are in process of disconnect and close - return brokerService.pulsar().getPulsarResources().getNamespaceResources() - .getPoliciesAsync(TopicName.get(topic).getNamespaceObject()) - .thenAcceptAsync(optPolicies -> { - if (optPolicies.isPresent()) { - if (optPolicies.get().replication_clusters != null) { - Set configuredClusters = Sets.newTreeSet(optPolicies.get().replication_clusters); - replicators.forEach((region, replicator) -> { - if (configuredClusters.contains(region)) { - replicator.startProducer(); - } - }); - } - } else { - replicators.forEach((region, replicator) -> replicator.startProducer()); - } - }, getOrderedExecutor()).exceptionally(ex -> { - if (log.isDebugEnabled()) { - log.debug("[{}] Error getting policies while starting repl-producers {}", topic, ex.getMessage()); - } - replicators.forEach((region, replicator) -> replicator.startProducer()); - return null; - }); - } - - public CompletableFuture stopReplProducers() { - List> closeFutures = new ArrayList<>(); - replicators.forEach((region, replicator) -> closeFutures.add(replicator.terminate())); - shadowReplicators.forEach((__, replicator) -> closeFutures.add(replicator.terminate())); - return FutureUtil.waitForAll(closeFutures); - } - - private synchronized CompletableFuture closeReplProducersIfNoBacklog() { - List> closeFutures = new ArrayList<>(); - replicators.forEach((region, replicator) -> closeFutures.add(replicator.disconnect())); - shadowReplicators.forEach((__, replicator) -> closeFutures.add(replicator.disconnect())); - return FutureUtil.waitForAll(closeFutures); - } - @Override protected void handleProducerRemoved(Producer producer) { super.handleProducerRemoved(producer); @@ -3285,9 +3242,9 @@ public boolean isActive(InactiveTopicDeleteMode deleteMode) { } if (TopicName.get(topic).isGlobal()) { // no local producers - return hasLocalProducers(); + return hasProducersActive() || hasActiveReplicators(); } else { - return currentUsageCount() != 0; + return currentUsageCount() != 0 || hasActiveReplicators(); } } @@ -3494,51 +3451,8 @@ public void checkGC() { // Topic activity is still within the retention period return; } else { - CompletableFuture replCloseFuture = new CompletableFuture<>(); - - if (TopicName.get(topic).isGlobal()) { - // For global namespace, close repl producers first. - // Once all repl producers are closed, we can delete the topic, - // provided no remote producers connected to the broker. - if (log.isDebugEnabled()) { - log.debug("[{}] Global topic inactive for {} seconds, closing repl producers.", topic, - maxInactiveDurationInSec); - } - /** - * There is a race condition that may cause a NPE: - * - task 1: a callback of "replicator.cursor.asyncRead" will trigger a replication. - * - task 2: "closeReplProducersIfNoBacklog" called by current thread will make the variable - * "replicator.producer" to a null value. - * Race condition: task 1 will get a NPE when it tries to send messages using the variable - * "replicator.producer", because task 2 will set this variable to "null". - * TODO Create a seperated PR to fix it. - */ - closeReplProducersIfNoBacklog().thenRun(() -> { - if (hasRemoteProducers()) { - if (log.isDebugEnabled()) { - log.debug("[{}] Global topic has connected remote producers. Not a candidate for GC", - topic); - } - replCloseFuture - .completeExceptionally(new TopicBusyException("Topic has connected remote producers")); - } else { - log.info("[{}] Global topic inactive for {} seconds, closed repl producers", topic, - maxInactiveDurationInSec); - replCloseFuture.complete(null); - } - }).exceptionally(e -> { - if (log.isDebugEnabled()) { - log.debug("[{}] Global topic has replication backlog. Not a candidate for GC", topic); - } - replCloseFuture.completeExceptionally(e.getCause()); - return null; - }); - } else { - replCloseFuture.complete(null); - } - - replCloseFuture.thenCompose(v -> delete(deleteMode == InactiveTopicDeleteMode.delete_when_no_subscriptions, - deleteMode == InactiveTopicDeleteMode.delete_when_subscriptions_caught_up, false)) + delete(deleteMode == InactiveTopicDeleteMode.delete_when_no_subscriptions, + deleteMode == InactiveTopicDeleteMode.delete_when_subscriptions_caught_up, false) .thenCompose((res) -> tryToDeletePartitionedMetadata()) .thenRun(() -> log.info("[{}] Topic deleted successfully due to inactivity", topic)) .exceptionally(e -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java index a156566d0feaa..f19570afcd6e8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java @@ -145,7 +145,6 @@ public void localSubscriptionUpdated(String subscriptionName, } private void receivedSnapshotRequest(ReplicatedSubscriptionsSnapshotRequest request) { - // if replicator producer is already closed, restart it to send snapshot response Replicator replicator = topic.getReplicators().get(request.getSourceCluster()); if (replicator == null) { log.warn("[{}] Received replicated subscription snapshot request {} from cluster {}, but no replicator is" @@ -153,9 +152,6 @@ private void receivedSnapshotRequest(ReplicatedSubscriptionsSnapshotRequest requ topic.getName(), request.getSnapshotId(), request.getSourceCluster()); return; } - if (!replicator.isConnected()) { - topic.startReplProducers(); - } // Send response containing the current last written message id. The response // marker we're publishing locally and then replicating will have a higher diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java index a5d0c6216f6d4..638a60a4cb0de 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ShadowReplicator.java @@ -61,8 +61,7 @@ protected boolean replicateEntries(List entries, InFlightTask inFlightTas try { // This flag is set to true when we skip at least one local message, // in order to skip remaining local messages. - boolean isLocalMessageSkippedOnce = false; - boolean skipRemainingMessages = inFlightTask.isSkipReadResultDueToCursorRewind(); + boolean skipRemainingMessages = false; for (int i = 0; i < entries.size(); i++) { Entry entry = entries.get(i); // Skip the messages since the replicator need to fetch the schema info to replicate the schema to the @@ -99,14 +98,14 @@ protected boolean replicateEntries(List entries, InFlightTask inFlightTas continue; } - if (STATE_UPDATER.get(this) != State.Started || isLocalMessageSkippedOnce) { + if (STATE_UPDATER.get(this) != State.Started || inFlightTask.isSkipReadResultDueToCursorRewind()) { // The producer is not ready yet after having stopped/restarted. Drop the message because it will // recovered when the producer is ready if (log.isDebugEnabled()) { log.debug("[{}] Dropping read message at {} because producer is not ready", replicatorId, entry.getPosition()); } - isLocalMessageSkippedOnce = true; + skipRemainingMessages = true; inFlightTask.incCompletedEntries(); entry.release(); msg.recycle(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java index f77615b77a84a..df1d18a00dac1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java @@ -54,6 +54,9 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -73,6 +76,7 @@ import org.apache.bookkeeper.mledger.impl.ManagedLedgerTest; import org.apache.commons.collections4.CollectionUtils; import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.resources.ClusterResources; import org.apache.pulsar.broker.service.nonpersistent.NonPersistentReplicator; import org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic; @@ -115,6 +119,8 @@ import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; +import org.apache.pulsar.zookeeper.ZookeeperServerTest; import org.awaitility.Awaitility; import org.awaitility.reflect.WhiteboxImpl; import org.glassfish.jersey.client.JerseyClient; @@ -142,6 +148,11 @@ public void cleanup() throws Exception { super.cleanup(); } + protected void setConfigDefaults(ServiceConfiguration config, String clusterName, + LocalBookkeeperEnsemble bookkeeperEnsemble, ZookeeperServerTest brokerConfigZk) { + super.setConfigDefaults(config, clusterName, bookkeeperEnsemble, brokerConfigZk); + } + @Test(timeOut = 45 * 1000) public void testReceiverSideReplicationStats() throws Exception { final String topic = BrokerTestUtil.newUniqueName("persistent://" + replicatedNamespace + "/tp_"); @@ -221,6 +232,189 @@ public void testDeleteTopicWhenReplicating() throws Exception { }); } + @DataProvider + public Object[][] paramsDisconnectReplicator() { + // Binary way replication. + // local producers on cluster-1 registered. + // local producers on cluster-1 have traffic. + // replicator producer from cluster-2 has traffic. + // replicator producer from cluster-2 is present. + return new Object[][] { + {true, true, false, true, true}, // verify-cluster-2: no replicator terminate occurs. + {true, true, false, false, true}, // verify-cluster-2: replicator terminated and resumed. + {true, true, false, false, false}, // verify-cluster-2: replicator terminated and resumed. + {true, false, false, true, true}, // verify-cluster-2: no replicator terminate occurs. + {true, false, false, false, true}, // verify-cluster-2: replicator terminated and resumed. + {true, false, false, false, false}, // verify-cluster-2: replicator terminated and resumed. + + {false, false, false, false, false}, // verify-cluster-2: replicator terminated and resumed. + {false, true, false, false, false} // verify-cluster-2: replicator terminated and resumed. + }; + } + + @Test(timeOut = 240 * 1000, dataProvider = "paramsDisconnectReplicator") + public void testDisconnectAndReconnectReplicator(boolean binaryWayRepl, + boolean hasLocalProducerRegistered, + boolean localProducerHasTraffic, + boolean hasRemoteProducerTraffic, + boolean hasRemoteProducerRegistered) throws Exception { + ScheduledExecutorService executor1 = Executors.newScheduledThreadPool(1); + ScheduledExecutorService executor2 = Executors.newScheduledThreadPool(1); + ScheduledFuture checkInactiveTopic = executor1.scheduleWithFixedDelay(() -> { + pulsar1.getBrokerService().checkInactiveReplication(); + }, 10, 10, TimeUnit.SECONDS); + // local cluster: let inactive replicator check faster. + int replicationInactiveThresholdSeconds1 = pulsar1.getConfig().getBrokerReplicationInactiveThresholdSeconds(); + pulsar1.getConfig().setBrokerReplicationInactiveThresholdSeconds(30); + // remote cluster: let inactive topic deletion never occur. + int replicationInactiveThresholdSeconds2 = pulsar2.getConfig().getBrokerReplicationInactiveThresholdSeconds(); + pulsar2.getConfig().setBrokerReplicationInactiveThresholdSeconds(3600 * 24); + // Lat topic GC does not execute. + int inactiveTopicsMaxInactiveDurationSeconds = pulsar1.getConfig() + .getBrokerDeleteInactiveTopicsMaxInactiveDurationSeconds(); + pulsar1.getConfig().setBrokerDeleteInactiveTopicsMaxInactiveDurationSeconds(3600 * 24); + + // Check params. + if (hasRemoteProducerTraffic && !hasRemoteProducerRegistered) { + throw new Exception("If has traffic from remote cluster, the param \"hasRemoteProducer\" can not be false"); + } + // Check params. + if (localProducerHasTraffic && !hasLocalProducerRegistered) { + throw new Exception("If has local traffic, the param \"localProducerEmpty\" can not be true"); + } + + ScheduledFuture scheduledPublish1 = null; + ScheduledFuture scheduledPublish2 = null; + final String topic = BrokerTestUtil.newUniqueName("persistent://" + replicatedNamespace + "/tp_"); + + // Init by params: local producers. + final Producer producer1A = client1.newProducer(Schema.STRING).topic(topic).create(); + Producer producer1B = null; + if (!hasLocalProducerRegistered) { + producer1A.close(); + } + // Init by params: local producer traffic. + if (localProducerHasTraffic) { + AtomicInteger msgCount = new AtomicInteger(); + scheduledPublish1 = executor1.scheduleWithFixedDelay(() -> { + producer1A.sendAsync(msgCount.incrementAndGet() + ""); + }, 1, 1, TimeUnit.SECONDS); + } + // Init by params: binary way replication. + waitReplicatorStarted(topic, pulsar2); + if (binaryWayRepl) { + admin2.topics().setReplicationClusters(topic, Arrays.asList(cluster1, cluster2)); + waitReplicatorStarted(topic, pulsar1); + } + final PersistentTopic persistentTopic1 = (PersistentTopic) broker1.getTopic(topic, false).join().get(); + final PersistentTopic persistentTopic2 = (PersistentTopic) broker2.getTopic(topic, false).join().get(); + // Init by params: remote producer traffic. + final Producer producer2 = client2.newProducer(Schema.STRING).topic(topic).create(); + if (hasRemoteProducerTraffic) { + AtomicInteger msgCount = new AtomicInteger(); + scheduledPublish2 = executor2.scheduleWithFixedDelay(() -> { + producer2.sendAsync(msgCount.incrementAndGet() + ""); + }, 1, 1, TimeUnit.SECONDS); + } + // Init by params: remote producers. + if (binaryWayRepl && !hasRemoteProducerTraffic && !hasRemoteProducerRegistered) { + persistentTopic2.getReplicators().get(cluster1).terminate(); + } + + // Verify: all states match params. + Thread.sleep(3000); + // All states match: local producers. + if (!hasLocalProducerRegistered) { + assertFalse(persistentTopic1.getProducers().values().stream() + .filter(p -> !p.isRemote()).findAny().isPresent()); + } else { + Optional serviceProducer1 = persistentTopic1.getProducers() + .values().stream().filter(p -> !p.isRemote()).findAny(); + assertTrue(serviceProducer1.isPresent()); + } + // All states match: remote producers. + if (binaryWayRepl) { + if (!hasRemoteProducerRegistered) { + assertFalse(persistentTopic1.getProducers().values().stream() + .filter(p -> p.isRemote()).findAny().isPresent()); + } else { + Optional serviceProducer1 = persistentTopic1.getProducers() + .values().stream().filter(p -> p.isRemote()).findAny(); + assertTrue(serviceProducer1.isPresent()); + } + } + + // Verify: replicator terminated or not. + if (hasRemoteProducerTraffic || localProducerHasTraffic) { + long verifyStartTime = System.currentTimeMillis(); + while (System.currentTimeMillis() - verifyStartTime < 100_000) { + assertFalse(persistentTopic1.getReplicators().isEmpty()); + PersistentReplicator persistentReplicator = + (PersistentReplicator) persistentTopic1.getReplicators().get(cluster2); + assertTrue(persistentReplicator.isConnected()); + assertEquals(persistentReplicator.getState(), AbstractReplicator.State.Started); + Thread.sleep(1000); + } + } else { + Thread.sleep(100_000); + assertFalse(persistentTopic1.getReplicators().isEmpty()); + PersistentReplicator persistentReplicatorA = + (PersistentReplicator) persistentTopic1.getReplicators().get(cluster2); + assertFalse(persistentReplicatorA.isConnected()); + assertEquals(persistentReplicatorA.getState(), AbstractReplicator.State.Disconnected); + + // Verify: resume. + if (hasRemoteProducerRegistered && !hasRemoteProducerTraffic) { + producer2.send("msg-remote"); + } + if (!hasLocalProducerRegistered) { + producer1B = client1.newProducer(Schema.STRING).topic(topic).create(); + producer1B.send("msg-local"); + } else { + producer1A.send("msg-local"); + } + Awaitility.await().untilAsserted(() -> { + assertFalse(persistentTopic1.getReplicators().isEmpty()); + PersistentReplicator persistentReplicatorB = + (PersistentReplicator) persistentTopic1.getReplicators().get(cluster2); + assertTrue(persistentReplicatorB.isConnected()); + assertEquals(persistentReplicatorB.getState(), AbstractReplicator.State.Started); + }); + } + + // cleanup. + pulsar1.getConfig().setBrokerReplicationInactiveThresholdSeconds(replicationInactiveThresholdSeconds1); + pulsar2.getConfig().setBrokerReplicationInactiveThresholdSeconds(replicationInactiveThresholdSeconds2); + pulsar1.getConfig().setBrokerDeleteInactiveTopicsMaxInactiveDurationSeconds( + inactiveTopicsMaxInactiveDurationSeconds); + if (scheduledPublish1 != null) { + scheduledPublish1.cancel(true); + } + if (scheduledPublish2 != null) { + scheduledPublish2.cancel(true); + } + checkInactiveTopic.cancel(true); + if (producer1A.isConnected()) { + producer1A.close(); + } + if (producer1B != null && producer1B.isConnected()) { + producer1B.close(); + } + if (producer2.isConnected()) { + producer2.close(); + } + if (binaryWayRepl) { + admin2.topics().setReplicationClusters(topic, Arrays.asList(cluster2)); + waitReplicatorStopped(pulsar2, pulsar1, topic); + } + cleanupTopics(() -> { + admin1.topics().delete(topic); + admin2.topics().delete(topic); + }); + executor1.shutdown(); + executor2.shutdown(); + } + @Test(timeOut = 45 * 1000) public void testReplicatorProducerStatInTopic() throws Exception { final String topicName = BrokerTestUtil.newUniqueName("persistent://" + replicatedNamespace + "/tp_"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTestBase.java index 0b033ec269cc2..da486b7d94991 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTestBase.java @@ -44,6 +44,7 @@ import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic; import org.apache.pulsar.broker.service.persistent.GeoPersistentReplicator; import org.apache.pulsar.broker.service.persistent.PersistentReplicator; import org.apache.pulsar.broker.service.persistent.PersistentTopic; @@ -55,6 +56,7 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.ProducerImpl; import org.apache.pulsar.common.naming.SystemTopicNames; +import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.policies.data.ClusterData; @@ -407,8 +409,13 @@ protected void waitReplicatorStarted(String topicName, PulsarService remoteClust Awaitility.await().untilAsserted(() -> { Optional topicOptional2 = remoteCluster.getBrokerService().getTopic(topicName, false).get(); assertTrue(topicOptional2.isPresent()); - PersistentTopic persistentTopic2 = (PersistentTopic) topicOptional2.get(); - assertFalse(persistentTopic2.getProducers().isEmpty()); + if (TopicName.get(topicName).getDomain().equals(TopicDomain.persistent)) { + PersistentTopic persistentTopic2 = (PersistentTopic) topicOptional2.get(); + assertFalse(persistentTopic2.getProducers().isEmpty()); + } else { + NonPersistentTopic nonPersistentTopic2 = (NonPersistentTopic) topicOptional2.get(); + assertFalse(nonPersistentTopic2.getProducers().isEmpty()); + } }); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalPartitionedTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalPartitionedTest.java index cec0f16187429..cdeeaba29506d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalPartitionedTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalPartitionedTest.java @@ -84,6 +84,16 @@ public void testDeleteTopicWhenReplicating() throws Exception { super.testDeleteTopicWhenReplicating(); } + @Test(enabled = false) + public void testDisconnectAndReconnectReplicator(boolean binaryWayRepl, + boolean hasLocalProducerRegistered, + boolean localProducerHasTraffic, + boolean hasRemoteProducerTraffic, + boolean hasRemoteProducerRegistered) throws Exception { + super.testDisconnectAndReconnectReplicator(binaryWayRepl, hasLocalProducerRegistered, localProducerHasTraffic, + hasRemoteProducerTraffic, hasRemoteProducerRegistered); + } + @Override @Test(enabled = false) public void testReplicatorProducerStatInTopic() throws Exception { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalZKTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalZKTest.java index 6c73f9548f068..63f199d59950f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalZKTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalZKTest.java @@ -54,6 +54,8 @@ import org.apache.pulsar.common.policies.data.AutoFailoverPolicyData; import org.apache.pulsar.common.policies.data.AutoFailoverPolicyType; import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride; +import org.apache.pulsar.common.policies.data.InactiveTopicDeleteMode; +import org.apache.pulsar.common.policies.data.InactiveTopicPolicies; import org.apache.pulsar.common.policies.data.NamespaceIsolationData; import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.TopicPolicies; @@ -810,6 +812,82 @@ public void testSystemTopicCreationWithDifferentTopicCreationRule(int localSyste admin2.topics().delete(tp, false); } + @Test(enabled = false) + public void testDisconnectAndReconnectReplicator(boolean binaryWayRepl, + boolean hasLocalProducerRegistered, + boolean localProducerHasTraffic, + boolean hasRemoteProducerTraffic, + boolean hasRemoteProducerRegistered) throws Exception { + super.testDisconnectAndReconnectReplicator(binaryWayRepl, hasLocalProducerRegistered, localProducerHasTraffic, + hasRemoteProducerTraffic, hasRemoteProducerRegistered); + } + + @Test + public void testTopicGCDoesNotDisconnectReplicatorWhenRemoteProducerIsActive() throws Exception { + int replicationInactiveThresholdSeconds = pulsar1.getConfig().getBrokerReplicationInactiveThresholdSeconds(); + pulsar1.getConfig().setBrokerReplicationInactiveThresholdSeconds(3600); + final String topic = BrokerTestUtil.newUniqueName("persistent://" + replicatedNamespace + "/tp_"); + admin1.topics().createNonPartitionedTopic(topic); + Producer producer1 = client1.newProducer(Schema.STRING).topic(topic).create(); + + try { + producer1.send("msg-1"); + waitReplicatorStarted(topic, pulsar1); + waitReplicatorStarted(topic, pulsar2); + PersistentTopic persistentTopic2 = (PersistentTopic) broker2.getTopic(topic, false) + .join().get(); + + // Set inactive policies. + InactiveTopicPolicies inactiveTopicPolicies = new InactiveTopicPolicies(); + inactiveTopicPolicies.setInactiveTopicDeleteMode(InactiveTopicDeleteMode.delete_when_no_subscriptions); + inactiveTopicPolicies.setMaxInactiveDurationSeconds(10); + inactiveTopicPolicies.setDeleteWhileInactive(true); + admin2.topicPolicies().setInactiveTopicPolicies(topic, inactiveTopicPolicies); + + // Ensure policies were set successfully. + Awaitility.await().untilAsserted(() -> { + assertFalse(persistentTopic2.getProducers().values().stream() + .anyMatch(producer -> !producer.isRemote())); + assertTrue(persistentTopic2.getSubscriptions().isEmpty()); + assertTrue(persistentTopic2.getInactiveTopicPolicies().isDeleteWhileInactive()); + assertEquals(persistentTopic2.getInactiveTopicPolicies().getMaxInactiveDurationSeconds(), 10); + + Replicator replicator = persistentTopic2.getReplicators().get(cluster1); + assertNotNull(replicator); + assertTrue(replicator.isConnected()); + assertEquals(replicator.getNumberOfEntriesInBacklog(), 0); + }); + + // Trigger GC. + persistentTopic2.disconnectReplicatorsIfNoTrafficAndBacklog(); + persistentTopic2.checkGC(); + Thread.sleep(15 * 1000); + persistentTopic2.disconnectReplicatorsIfNoTrafficAndBacklog(); + persistentTopic2.checkGC(); + + // Verify: the replication is not disconnected due to Topic GC. + Replicator replicator = persistentTopic2.getReplicators().get(cluster1); + assertNotNull(replicator); + assertTrue(replicator.isConnected()); + + // Verify: the replication still works. + producer1.send("msg-2"); + Awaitility.await().untilAsserted(() -> { + assertEquals(admin2.topics().getStats(topic).getReplication().get(cluster1).getReplicationBacklog(), 0); + }); + + } finally { + pulsar1.getConfig().setBrokerReplicationInactiveThresholdSeconds(replicationInactiveThresholdSeconds); + producer1.close(); + admin1.topics().setReplicationClusters(topic, Arrays.asList(cluster1)); + admin2.topics().setReplicationClusters(topic, Arrays.asList(cluster2)); + waitReplicatorStopped(topic, pulsar1, pulsar2, false); + waitReplicatorStopped(topic, pulsar2, pulsar1, false); + admin1.topics().delete(topic, false); + admin2.topics().delete(topic, false); + } + } + @Test public void testUpdateNamespacePolicies() throws Exception { // Create a namespace and allow both clusters to access. diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java index 19ccc9d4ec533..70a027cdd625b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java @@ -1664,17 +1664,19 @@ public void testFailoverSubscription() throws Exception { } /** - * NonPersistentReplicator.removeReplicator doesn't remove replicator in atomic way and does in multiple step: - * 1. disconnect replicator producer + * PersistentReplicator.removeReplicator doesn't remove replicator in atomic way and does in multiple step: + * 1. Turn off replication. *

- * 2. close cursor + * 2. Broker will do two things: + * 2-1. terminate replication. + * 2-2. delete cursor that named "repl.x" *

- * 3. remove from replicator-list. + * 3. remove the terminated replicator from "topic.replicators". *

* - * If we try to startReplicationProducer before step-c finish then it should not avoid restarting repl-producer. - * - * @throws Exception + * Test: + * do: try to restart replicator producer before step-2-2 finish. + * verify: the replicator producer will not be started. */ @Test public void testAtomicReplicationRemoval() throws Exception { @@ -1724,9 +1726,14 @@ public CompletableFuture createAsync() { // step-2 now, policies doesn't have removed replication cluster so, it should not invoke "startProducer" of the // replicator // try to start replicator again - topic.startReplProducers().join(); + Awaitility.await().untilAsserted(() -> { + assertEquals(replicator.getState(), AbstractReplicator.State.Terminated); + }); + replicator.startProducer(); + Thread.sleep(10_000); // verify: replicator.startProducer is not invoked - verify(replicator, Mockito.times(1)).startProducer(); + assertEquals(replicator.getState(), AbstractReplicator.State.Terminated); + assertFalse(replicator.isConnected()); // step-3 : complete the callback to remove replicator from the list ArgumentCaptor captor = ArgumentCaptor.forClass(DeleteCursorCallback.class); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicationTopicGcTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicationTopicGcTest.java index c2b6d4281a9b0..c72d2bcef53d0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicationTopicGcTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicationTopicGcTest.java @@ -28,6 +28,8 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.service.persistent.GeoPersistentReplicator; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; @@ -77,6 +79,7 @@ protected void setConfigDefaults(ServiceConfiguration config, String clusterName config.setBrokerDeleteInactiveTopicsFrequencySeconds(5); config.setBrokerDeleteInactiveTopicsMaxInactiveDurationSeconds(5); config.setReplicationPolicyCheckDurationSeconds(1); + config.setBrokerReplicationInactiveThresholdSeconds(5); } @Test(dataProvider = "topicTypes") @@ -144,6 +147,7 @@ public void testRemoteClusterStillConsumeAfterCurrentClusterGc(TopicType topicTy // Wait for replicator started. Producer producer1 = client1.newProducer(Schema.STRING).topic(topicName).create(); waitReplicatorStarted(subTopic); + PersistentTopic persistentTopic1 = (PersistentTopic) broker1.getTopic(subTopic, false).get().get(); admin2.topics().createSubscription(topicName, subscription, MessageId.earliest); if (usingGlobalZK) { @@ -159,6 +163,10 @@ public void testRemoteClusterStillConsumeAfterCurrentClusterGc(TopicType topicTy // Trigger GC through close all clients. producer1.close(); + // Manually skip the check "brokerReplicationInactiveThresholdSeconds". + GeoPersistentReplicator geoPersistentReplicator1 = + (GeoPersistentReplicator) persistentTopic1.getReplicators().get(cluster2); + geoPersistentReplicator1.disconnect(); // Verify: the topic was removed on the source cluster. Awaitility.await().atMost(60, TimeUnit.SECONDS).untilAsserted(() -> { // sub topic. diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java index 0426bc3656aec..0b19e948125ed 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java @@ -71,7 +71,9 @@ import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException; +import org.apache.pulsar.broker.service.persistent.GeoPersistentReplicator; import org.apache.pulsar.broker.service.persistent.PersistentReplicator; +import org.apache.pulsar.broker.service.persistent.PersistentReplicatorInflightTaskTest; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.stats.OpenTelemetryReplicatorStats; import org.apache.pulsar.client.admin.PulsarAdmin; @@ -1827,21 +1829,26 @@ public void testReplicatorWithTTL() throws Exception { waitReplicateFinish(topic, admin1); // Pause replicator + List resumeReplicatorFunctions = new ArrayList<>(); persistentTopic.getReplicators().forEach((cluster, replicator) -> { PersistentReplicator persistentReplicator = (PersistentReplicator) replicator; - pauseReplicator(persistentReplicator); + resumeReplicatorFunctions.add(PersistentReplicatorInflightTaskTest.pauseReplicator(persistentReplicator)); }); // Send V2 and V3 messages, then let them expire. These messages will not be replicated to the remote cluster. persistentProducer1.send("V2".getBytes()); persistentProducer1.send("V3".getBytes()); - Thread.sleep(1000); - admin1.topics().expireMessagesForAllSubscriptions(topic.toString(), 1); + Thread.sleep(2000); + GeoPersistentReplicator persistentReplicator = + (GeoPersistentReplicator) persistentTopic.getReplicators().values().iterator().next(); + persistentReplicator.expireMessages(1); + waitReplicateFinish(topic, admin1); // Start replicator + for (Runnable r : resumeReplicatorFunctions) { + r.run(); + } persistentTopic.getReplicators().forEach((cluster, replicator) -> { - PersistentReplicator persistentReplicator = (PersistentReplicator) replicator; - resumeReplicator(persistentReplicator); Awaitility.await().untilAsserted(() -> { CompletableFuture> topic2 = pulsar2.getBrokerService().getTopic(topic.toString(), false); @@ -1918,7 +1925,7 @@ public void testReplicationMetrics() throws Exception { .stream() .map(PersistentReplicator.class::cast) .toList(); - persistentReplicators.forEach(this::pauseReplicator); + persistentReplicators.forEach(PersistentReplicatorInflightTaskTest::pauseReplicator); producer1.produce(5); Awaitility.await().untilAsserted(() -> { persistentReplicators.forEach(repl -> repl.expireMessages(1)); @@ -1998,18 +2005,4 @@ public void testEnableReplicationWithNamespaceAllowedClustersPolices() throws Ex assertTrue(replicator.isConnected()); }); } - - private void pauseReplicator(PersistentReplicator replicator) { - Awaitility.await().untilAsserted(() -> { - assertTrue(replicator.isConnected()); - }); - Awaitility.await().until(() -> { - replicator.disconnect().join(); - return true; - }); - } - - private void resumeReplicator(PersistentReplicator replicator) { - replicator.startProducer(); - } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java index 0c22b14acf748..83529969df417 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java @@ -496,4 +496,17 @@ public void testAcquirePermitsIfNotFetchingSchema() throws Exception { inFlightTasks.addAll(originalTasks); } } + + public static Runnable pauseReplicator(PersistentReplicator replicator) { + Awaitility.await().untilAsserted(() -> { + assertTrue(replicator.isConnected()); + }); + replicator.beforeTerminateOrCursorRewinding(PersistentReplicator.ReasonOfWaitForCursorRewinding.Disconnecting); + replicator.doRewindCursor(false); + InFlightTask inFlightTask = replicator.createOrRecycleInFlightTaskIntoQueue(PositionFactory.create(1, 1), 1); + return () -> { + inFlightTask.setEntries(Collections.emptyList()); + replicator.readMoreEntries(); + }; + } } From b3210140458fc46daf038cfe099f2569ef38cf2c Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Fri, 26 Jun 2026 19:28:42 +0800 Subject: [PATCH 078/213] [fix][sec][branch-4.0] Upgrade Jackson version to 2.18.8 (#26098) --- .../server/src/assemble/LICENSE.bin.txt | 22 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 22 +++++++++---------- pom.xml | 2 +- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 611f3f35b4bd8..1fe9c7e1d520a 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -249,17 +249,17 @@ The Apache Software License, Version 2.0 - info.picocli-picocli-shell-jline3-4.7.7.jar * High Performance Primitive Collections for Java -- com.carrotsearch-hppc-0.9.1.jar * Jackson - - com.fasterxml.jackson.core-jackson-annotations-2.18.6.jar - - com.fasterxml.jackson.core-jackson-core-2.18.6.jar - - com.fasterxml.jackson.core-jackson-databind-2.18.6.jar - - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.18.6.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.18.6.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.18.6.jar - - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.18.6.jar - - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.18.6.jar - - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.18.6.jar - - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.18.6.jar - - com.fasterxml.jackson.module-jackson-module-parameter-names-2.18.6.jar + - com.fasterxml.jackson.core-jackson-annotations-2.18.8.jar + - com.fasterxml.jackson.core-jackson-core-2.18.8.jar + - com.fasterxml.jackson.core-jackson-databind-2.18.8.jar + - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.18.8.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.18.8.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.18.8.jar + - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.18.8.jar + - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.18.8.jar + - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.18.8.jar + - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.18.8.jar + - com.fasterxml.jackson.module-jackson-module-parameter-names-2.18.8.jar * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.9.1.jar * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar * Fastutil -- it.unimi.dsi-fastutil-8.5.16.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 8b610881d1fd8..5b0ff548854a8 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -313,17 +313,17 @@ The Apache Software License, Version 2.0 - picocli-4.7.7.jar - picocli-shell-jline3-4.7.7.jar * Jackson - - jackson-annotations-2.18.6.jar - - jackson-core-2.18.6.jar - - jackson-databind-2.18.6.jar - - jackson-dataformat-yaml-2.18.6.jar - - jackson-jaxrs-base-2.18.6.jar - - jackson-jaxrs-json-provider-2.18.6.jar - - jackson-module-jaxb-annotations-2.18.6.jar - - jackson-module-jsonSchema-2.18.6.jar - - jackson-datatype-jdk8-2.18.6.jar - - jackson-datatype-jsr310-2.18.6.jar - - jackson-module-parameter-names-2.18.6.jar + - jackson-annotations-2.18.8.jar + - jackson-core-2.18.8.jar + - jackson-databind-2.18.8.jar + - jackson-dataformat-yaml-2.18.8.jar + - jackson-jaxrs-base-2.18.8.jar + - jackson-jaxrs-json-provider-2.18.8.jar + - jackson-module-jaxb-annotations-2.18.8.jar + - jackson-module-jsonSchema-2.18.8.jar + - jackson-datatype-jdk8-2.18.8.jar + - jackson-datatype-jsr310-2.18.8.jar + - jackson-module-parameter-names-2.18.8.jar * Caffeine -- caffeine-2.9.1.jar * Conscrypt -- conscrypt-openjdk-uber-2.5.2.jar * Gson diff --git a/pom.xml b/pom.xml index 129d1efb5893b..111d31c130516 100644 --- a/pom.xml +++ b/pom.xml @@ -209,7 +209,7 @@ flexible messaging model and an intuitive client API. 2.0.11 2.0.6 2.0.1 - 2.18.6 + 2.18.8 8.5.16 0.10.2 1.6.2 From be4b3878c5fa868d3d7bbe28d10fb16e12155116 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=81=93=E5=90=9B-=20Tao=20Jiuming?= Date: Tue, 23 Jun 2026 15:26:13 +0800 Subject: [PATCH 079/213] [improve][broker] Trim orphaned bucket snapshots when ledgers are deleted (#25984) (cherry picked from commit e45b425f927c63b4de4093ee6f9f0218b188b701) --- .../bucket/BucketDelayedDeliveryTracker.java | 110 ++++++++- .../BucketDelayedDeliveryTrackerTest.java | 213 ++++++++++++++++++ 2 files changed, 312 insertions(+), 11 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index 77c4798232b5d..f985cbdf34fdf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -38,6 +38,7 @@ import java.util.Optional; import java.util.TreeSet; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -48,6 +49,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.commons.collections4.CollectionUtils; @@ -112,6 +114,8 @@ public static record SnapshotKey(long ledgerId, long entryId) {} private CompletableFuture pendingLoad = null; + private volatile CompletableFuture trimFuture; + public BucketDelayedDeliveryTracker(AbstractPersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis, boolean isDelayedDeliveryDeliverAtTimeStrict, @@ -387,8 +391,15 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver afterCreateImmutableBucket(immutableBucketDelayedIndexPair, createStartTime); lastMutableBucket.resetLastMutableBucketRange(); - if (maxNumBuckets > 0 && immutableBuckets.asMapOfRanges().size() > maxNumBuckets) { - asyncMergeBucketSnapshot(); + if (maxNumBuckets > 0 && immutableBuckets.asMapOfRanges().size() > maxNumBuckets + && (trimFuture == null || trimFuture.isDone())) { + trimFuture = asyncTrimImmutableBuckets() + .thenCompose(ignore -> asyncMergeBucketSnapshot()) + .whenComplete((ignore, t) -> { + if (t != null) { + log.warn("Failed to trim or merge bucket snapshots", t); + } + }); } } @@ -452,6 +463,10 @@ private synchronized List selectMergedBuckets(final List asyncMergeBucketSnapshot() { List immutableBucketList = immutableBuckets.asMapOfRanges().values().stream().toList(); + if (maxNumBuckets <= 0 || immutableBucketList.size() <= maxNumBuckets) { + return CompletableFuture.completedFuture(null); + } + List toBeMergeImmutableBuckets = selectMergedBuckets(immutableBucketList, MAX_MERGE_NUM); if (toBeMergeImmutableBuckets.isEmpty()) { @@ -605,6 +620,7 @@ public synchronized NavigableSet getScheduledMessages(int maxMessages) } long cutoffTime = getCutoffTime(); + Long firstLiveLedgerId = firstActiveLedgerId(); lastMutableBucket.moveScheduledMessageToSharedQueue(cutoffTime, sharedBucketPriorityQueue); @@ -613,13 +629,19 @@ public synchronized NavigableSet getScheduledMessages(int maxMessages) while (n > 0 && !sharedBucketPriorityQueue.isEmpty()) { long timestamp = sharedBucketPriorityQueue.peekN1(); + long ledgerId = sharedBucketPriorityQueue.peekN2(); + long entryId = sharedBucketPriorityQueue.peekN3(); + if (firstLiveLedgerId != null && ledgerId < firstLiveLedgerId) { + sharedBucketPriorityQueue.pop(); + if (removeIndexBit(ledgerId, entryId)) { + numberDelayedMessages.decrementAndGet(); + } + continue; + } if (timestamp > cutoffTime) { break; } - long ledgerId = sharedBucketPriorityQueue.peekN2(); - long entryId = sharedBucketPriorityQueue.peekN3(); - SnapshotKey snapshotKey = new SnapshotKey(ledgerId, entryId); ImmutableBucket bucket = snapshotSegmentLastIndexMap.get(snapshotKey); @@ -724,12 +746,26 @@ public boolean shouldPauseAllDeliveries() { @Override public synchronized CompletableFuture clear() { - CompletableFuture future = cleanImmutableBuckets(); - sharedBucketPriorityQueue.clear(); - lastMutableBucket.clear(); - snapshotSegmentLastIndexMap.clear(); - numberDelayedMessages.set(0); - return future; + // Wait for any in-flight trim+merge to settle, then clear. + // Reuse trimFuture to block new triggers until the clear chain completes. + CompletableFuture before = trimFuture != null && !trimFuture.isDone() + ? trimFuture : CompletableFuture.completedFuture(null); + trimFuture = before + .exceptionally(t -> { + log.warn("Trim/merge buckets failed, but still clear", t); + return null; + }) + .thenCompose(__ -> { + synchronized (BucketDelayedDeliveryTracker.this) { + CompletableFuture future = cleanImmutableBuckets(); + sharedBucketPriorityQueue.clear(); + lastMutableBucket.clear(); + snapshotSegmentLastIndexMap.clear(); + numberDelayedMessages.set(0); + return future; + } + }); + return trimFuture; } @Override @@ -796,4 +832,56 @@ public Map genTopicMetricMap() { stats.recordBucketSnapshotSizeBytes(totalSnapshotLength.longValue()); return stats.genTopicMetricMap(); } + + /** + * Delete orphaned bucket snapshots whose ledger range lies entirely before the earliest + * surviving ledger. Buckets are deleted sequentially; the chain stops on first failure + * to avoid wasted work when storage is unavailable. + */ + private synchronized CompletableFuture asyncTrimImmutableBuckets() { + Long firstLedgerId = firstActiveLedgerId(); + if (null == firstLedgerId) { + return CompletableFuture.completedFuture(null); + } + ManagedLedger ledger = dispatcher.getCursor().getManagedLedger(); + + Map, ImmutableBucket> toBeDeletedBuckets = + new HashMap<>(immutableBuckets.subRangeMap(Range.lessThan(firstLedgerId)).asMapOfRanges()); + + if (toBeDeletedBuckets.isEmpty()) { + return CompletableFuture.completedFuture(null); + } + + String ledgerName = ledger.getName(); + CompletableFuture chain = CompletableFuture.completedFuture(null); + for (Map.Entry, ImmutableBucket> entry : toBeDeletedBuckets.entrySet()) { + chain = chain.thenCompose(__ -> + deleteBucketSnapshot(ledgerName, entry.getKey(), entry.getValue())); + } + return chain; + } + + private CompletableFuture deleteBucketSnapshot(String ledgerName, + Range range, ImmutableBucket bucket) { + return bucket.asyncDeleteBucketSnapshot(stats) + .handle((__, t) -> { + if (t != null) { + log.warn("Failed to delete bucket snapshot, LedgerName: {}, BucketKey: {}", + ledgerName, bucket.bucketKey()); + throw new CompletionException(t); + } + synchronized (this) { + snapshotSegmentLastIndexMap.entrySet().removeIf(entry -> entry.getValue() == bucket); + immutableBuckets.remove(range); + numberDelayedMessages.addAndGet(-bucket.getNumberBucketDelayedMessages()); + } + return null; + }); + } + + private Long firstActiveLedgerId() { + ManagedCursor cursor = dispatcher.getCursor(); + Position mdp = cursor.getMarkDeletedPosition(); + return mdp == null ? null : mdp.getLedgerId(); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java index 6ff98fa7f7004..802132480402c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java @@ -40,13 +40,16 @@ import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.bookkeeper.mledger.proto.MLDataFormats; import org.apache.commons.lang3.mutable.MutableLong; import org.apache.pulsar.broker.delayed.AbstractDeliveryTrackerTest; import org.apache.pulsar.broker.delayed.MockBucketSnapshotStorage; @@ -463,4 +466,214 @@ public void testClear(BucketDelayedDeliveryTracker tracker) tracker.close(); } + + private static class TrackerWithStorage { + final BucketDelayedDeliveryTracker tracker; + final MockBucketSnapshotStorage storage; + final AtomicLong clockTime; + + TrackerWithStorage(BucketDelayedDeliveryTracker tracker, MockBucketSnapshotStorage storage, + AtomicLong clockTime) { + this.tracker = tracker; + this.storage = storage; + this.clockTime = clockTime; + } + + void close() throws Exception { + tracker.close(); + storage.close(); + } + } + + private static class BlockingDeleteStorage extends MockBucketSnapshotStorage { + final CompletableFuture firstDeleteFuture = new CompletableFuture<>(); + final AtomicLong deleteCalls = new AtomicLong(); + + @Override + public CompletableFuture deleteBucketSnapshot(long bucketId) { + if (deleteCalls.incrementAndGet() <= 4) { + return firstDeleteFuture; + } + return super.deleteBucketSnapshot(bucketId); + } + } + + private TrackerWithStorage createTrackerWithMockLedger(long firstLedgerId, int maxNumBuckets) + throws Exception { + return createTrackerWithMockLedger(firstLedgerId, maxNumBuckets, new MockBucketSnapshotStorage()); + } + + private TrackerWithStorage createTrackerWithMockLedger(long firstLedgerId, int maxNumBuckets, + MockBucketSnapshotStorage storage) + throws Exception { + storage.start(); + + ManagedLedger mockLedger = mock(ManagedLedger.class); + NavigableMap ledgerInfo = new TreeMap<>(); + ledgerInfo.put(firstLedgerId, mock(MLDataFormats.ManagedLedgerInfo.LedgerInfo.class)); + when(mockLedger.getLedgersInfo()).thenReturn(ledgerInfo); + when(mockLedger.getName()).thenReturn("test-ledger"); + + ManagedCursor mockCursor = new MockManagedCursor("test-cursor") { + @Override + public ManagedLedger getManagedLedger() { + return mockLedger; + } + + @Override + public Position getMarkDeletedPosition() { + return PositionFactory.create(firstLedgerId, -1); + } + }; + + AbstractPersistentDispatcherMultipleConsumers disp = + mock(AbstractPersistentDispatcherMultipleConsumers.class); + Clock mockClock = mock(Clock.class); + AtomicLong mockClockTime = new AtomicLong(); + when(mockClock.millis()).then(x -> mockClockTime.get()); + doReturn(mockCursor).when(disp).getCursor(); + doReturn("persistent://public/default/testDelay" + " / " + mockCursor.getName()).when(disp).getName(); + + BucketDelayedDeliveryTracker tracker = new BucketDelayedDeliveryTracker(disp, mock(Timer.class), + 100000, mockClock, true, storage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, maxNumBuckets); + return new TrackerWithStorage(tracker, storage, mockClockTime); + } + + @Test + public void testTrimRemovesOrphanedBuckets() throws Exception { + long firstLedgerId = 31L; + int messageCount = 36; + TrackerWithStorage ts = createTrackerWithMockLedger(firstLedgerId, 5); + + for (int i = 1; i <= messageCount; i++) { + ts.tracker.addMessage(i, i, i * 10); + } + Awaitility.await().untilAsserted(() -> + Assert.assertTrue(ts.tracker.getImmutableBuckets().asMapOfRanges().values().stream() + .noneMatch(x -> x.merging))); + + int bucketCount = ts.tracker.getImmutableBuckets().asMapOfRanges().size(); + assertTrue(bucketCount <= 5, + "Bucket count " + bucketCount + " should be <= maxNumBuckets=5 after trim+merge"); + + ts.tracker.getImmutableBuckets().asMapOfRanges().forEach((range, bucket) -> + assertTrue(range.lowerEndpoint() >= firstLedgerId, + "Remaining bucket range " + range + " should be >= " + firstLedgerId)); + + long messagesAfterTrim = ts.tracker.getNumberOfDelayedMessages(); + ts.clockTime.set(messageCount * 10); + NavigableSet scheduledMessages = ts.tracker.getScheduledMessages(1); + assertTrue(scheduledMessages.stream().noneMatch(position -> position.getLedgerId() < firstLedgerId), + "Trimmed ledgers should not be returned from the loaded shared queue"); + assertEquals(ts.tracker.getNumberOfDelayedMessages(), messagesAfterTrim - scheduledMessages.size()); + + ts.close(); + } + + @Test + public void testTrimHandlesDeleteFailure() throws Exception { + long firstLedgerId = 50L; + int messageCount = 31; + TrackerWithStorage ts = createTrackerWithMockLedger(firstLedgerId, 5); + + // MaxRetryTimes=3 means the first trim delete attempt plus 3 retries = 4 exceptions consumed. + for (int i = 0; i < 4; i++) { + ts.storage.injectDeleteException( + new BucketSnapshotPersistenceException("Delete failed")); + } + + for (int i = 1; i <= messageCount; i++) { + ts.tracker.addMessage(i, i, i * 10); + } + Awaitility.await().untilAsserted(() -> + Assert.assertTrue(ts.tracker.getImmutableBuckets().asMapOfRanges().values().stream() + .noneMatch(x -> x.merging))); + + Awaitility.await().untilAsserted(() -> + assertTrue(ts.storage.deleteExceptionQueue.isEmpty(), + "Delete exception should have been consumed")); + + // Trim failed on the first orphaned bucket; the sequential chain stopped, so all + // 6 orphaned buckets remain in immutableBuckets. + assertTrue(ts.tracker.getImmutableBuckets().asMapOfRanges().size() > 0, + "Orphaned buckets should remain when trim delete fails"); + ts.tracker.getImmutableBuckets().asMapOfRanges().forEach((range, bucket) -> + assertTrue(range.upperEndpoint() < firstLedgerId, + "Remaining bucket " + range + " should be an orphaned bucket")); + + // numberDelayedMessages is unchanged because failed deletes do not decrement the count. + assertEquals(ts.tracker.getNumberOfDelayedMessages(), messageCount); + + ts.close(); + } + + @Test + public void testClearRunsAfterInFlightTrimFailure() throws Exception { + long firstLedgerId = 50L; + int messageCount = 31; + BlockingDeleteStorage storage = new BlockingDeleteStorage(); + TrackerWithStorage ts = createTrackerWithMockLedger(firstLedgerId, 5, storage); + + for (int i = 1; i <= messageCount; i++) { + ts.tracker.addMessage(i, i, i * 10); + } + Awaitility.await().untilAsserted(() -> + assertTrue(storage.deleteCalls.get() > 0, "Trim delete should be in flight")); + + CompletableFuture clearFuture = ts.tracker.clear(); + storage.firstDeleteFuture.completeExceptionally(new BucketSnapshotPersistenceException("Delete failed")); + + clearFuture.get(1, TimeUnit.MINUTES); + assertEquals(ts.tracker.getNumberOfDelayedMessages(), 0); + assertEquals(ts.tracker.getImmutableBuckets().asMapOfRanges().size(), 0); + assertEquals(ts.tracker.getLastMutableBucket().size(), 0); + assertEquals(ts.tracker.getSharedBucketPriorityQueue().size(), 0); + + ts.close(); + } + + @Test + public void testTrimWithNoOrphanedBuckets() throws Exception { + TrackerWithStorage ts = createTrackerWithMockLedger(0L, 5); + + for (int i = 1; i <= 31; i++) { + ts.tracker.addMessage(i, i, i * 10); + } + Awaitility.await().untilAsserted(() -> + Assert.assertTrue(ts.tracker.getImmutableBuckets().asMapOfRanges().values().stream() + .noneMatch(x -> x.merging))); + + int bucketCount = ts.tracker.getImmutableBuckets().asMapOfRanges().size(); + assertTrue(bucketCount <= 5, + "Bucket count " + bucketCount + " should be <= maxNumBuckets=5"); + assertTrue(bucketCount > 0, "Should have at least one bucket after merge"); + + ts.close(); + } + + @Test + public void testMergeEarlyReturnWhenWithinLimit() throws Exception { + TrackerWithStorage ts = createTrackerWithMockLedger(0L, 50); + + for (int i = 1; i <= 30; i++) { + ts.tracker.addMessage(i, i, i * 10); + } + Awaitility.await().untilAsserted(() -> + Assert.assertTrue(ts.tracker.getImmutableBuckets().asMapOfRanges().values().stream() + .noneMatch(x -> x.merging))); + + int bucketCount = ts.tracker.getImmutableBuckets().asMapOfRanges().size(); + assertTrue(bucketCount < 50, + "Bucket count " + bucketCount + " should be well below maxNumBuckets=50"); + + long msgsBefore = ts.tracker.getNumberOfDelayedMessages(); + ts.tracker.addMessage(200, 200, 200 * 10); + Awaitility.await().untilAsserted(() -> + Assert.assertTrue(ts.tracker.getImmutableBuckets().asMapOfRanges().values().stream() + .noneMatch(x -> x.merging))); + + assertEquals(ts.tracker.getNumberOfDelayedMessages(), msgsBefore + 1); + + ts.close(); + } } From af42891ff4599d47fa63abf359072d919ac58b5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=81=93=E5=90=9B-=20Tao=20Jiuming?= Date: Wed, 24 Jun 2026 18:02:51 +0800 Subject: [PATCH 080/213] [fix][broker] Guard BucketDelayedDeliveryTracker.nextDeliveryTime against empty queues (#26080) (cherry picked from commit 51e2b9a6bc9f739d3fc33b24821df15c588d4055) --- .../bucket/BucketDelayedDeliveryTracker.java | 5 ++ .../BucketDelayedDeliveryTrackerTest.java | 46 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index f985cbdf34fdf..6cf2876d4b880 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -593,6 +593,11 @@ protected synchronized long nextDeliveryTime() { return sharedBucketPriorityQueue.peekN1(); } else if (sharedBucketPriorityQueue.isEmpty() && !lastMutableBucket.isEmpty()) { return lastMutableBucket.nextDeliveryTime(); + } else if (lastMutableBucket.isEmpty() && sharedBucketPriorityQueue.isEmpty()) { + // numberDelayedMessages can be > 0 while both queues are empty (e.g. remaining + // messages live in not-yet-loaded snapshot segments). Returning Long.MAX_VALUE + // signals "no imminent delivery" without throwing on the empty queues. + return Long.MAX_VALUE; } long timestamp = lastMutableBucket.nextDeliveryTime(); long bucketTimestamp = sharedBucketPriorityQueue.peekN1(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java index 802132480402c..cfe7ed370daf9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java @@ -676,4 +676,50 @@ public void testMergeEarlyReturnWhenWithinLimit() throws Exception { ts.close(); } + + @Test + public void testGetScheduledMessagesWhenAllOrphaned() throws Exception { + // Reproduces IAE in nextDeliveryTime: when every delayed message lies below the + // mark-delete position, the filter in getScheduledMessages pops the in-memory + // messages without returning them. If the immutable bucket has additional messages + // still in storage (later snapshot segments), numberDelayedMessages stays > 0 + // while both the mutable bucket and the shared priority queue are empty. + // The trailing updateTimer -> nextDeliveryTime must not throw. + long firstLedgerId = 50L; + TrackerWithStorage ts = createTrackerWithMockLedger(firstLedgerId, 50); + + // Five delayed messages on the same orphaned ledger (ledgerId < firstLedgerId). + // They share a mutable bucket because seal requires a strictly greater ledgerId. + // Timestamps are 100ms apart so each lands in its own snapshot segment + // (timeStep=10ms); only the first segment is loaded into the shared queue at seal. + for (int i = 1; i <= 5; i++) { + ts.tracker.addMessage(1, i, i * 100); + } + // A new orphaned ledgerId triggers the seal, producing immutable bucket [1..1] + // with 5 messages across 5 segments; shared queue holds just the first segment. + ts.tracker.addMessage(2, 1, 600); + + Awaitility.await().untilAsserted(() -> + Assert.assertTrue(ts.tracker.getImmutableBuckets().asMapOfRanges().values().stream() + .noneMatch(x -> x.merging))); + + // In strict deliver-at mode getCutoffTime() is just clock.millis(), so advancing the + // clock past the trigger message's deliverAt (600) is enough for + // moveScheduledMessageToSharedQueue to flush the mutable bucket into the shared queue. + ts.clockTime.set(700); + + // Both queues end up empty (filter pops the two in-memory messages), but + // numberDelayedMessages is still 4 (segments 2..5 remain in storage). + NavigableSet scheduledMessages = ts.tracker.getScheduledMessages(10); + assertTrue(scheduledMessages.isEmpty(), + "Orphaned messages should be filtered out, not returned"); + assertTrue(ts.tracker.getNumberOfDelayedMessages() > 0, + "Remaining storage-only messages should keep the counter > 0"); + + // hasMessageAvailable calls nextDeliveryTime while numberDelayedMessages > 0; + // it must not throw IAE. + assertFalse(ts.tracker.hasMessageAvailable()); + + ts.close(); + } } From 64088b09a1cc0fbb1c3560fc99436bd1e60111c9 Mon Sep 17 00:00:00 2001 From: void-ptr974 Date: Thu, 25 Jun 2026 03:34:33 +0800 Subject: [PATCH 081/213] [improve][broker] Improve dispatch performance by summing entry bytes with a loop (#26055) (cherry picked from commit b2f37560b356a011bcd9bdb7d37d8ce01d95ea98) --- ...PersistentDispatcherMultipleConsumers.java | 10 +++ ...PersistentDispatcherMultipleConsumers.java | 2 +- ...entDispatcherMultipleConsumersClassic.java | 2 +- .../PersistentDispatcherTotalBytesTest.java | 78 +++++++++++++++++++ 4 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherTotalBytesTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/AbstractPersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/AbstractPersistentDispatcherMultipleConsumers.java index 79d365b9fee21..3a63dc6594704 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/AbstractPersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/AbstractPersistentDispatcherMultipleConsumers.java @@ -18,8 +18,10 @@ */ package org.apache.pulsar.broker.service.persistent; +import java.util.List; import java.util.Map; import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.AbstractDispatcherMultipleConsumers; @@ -64,4 +66,12 @@ public AbstractPersistentDispatcherMultipleConsumers(Subscription subscription, public abstract Map getBucketDelayedIndexStats(); public abstract boolean isClassic(); + + static long getTotalBytesSize(List entries) { + long totalBytesSize = 0; + for (int i = 0, entriesSize = entries.size(); i < entriesSize; i++) { + totalBytesSize += entries.get(i).getLength(); + } + return totalBytesSize; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index e51f8a3acf8fc..b85f3508d3dea 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -707,7 +707,7 @@ public final synchronized void readEntriesComplete(List entries, Object c log.debug("[{}] Distributing {} messages to {} consumers", name, entries.size(), consumerList.size()); } - long totalBytesSize = entries.stream().mapToLong(Entry::getLength).sum(); + long totalBytesSize = getTotalBytesSize(entries); updatePendingBytesToDispatch(totalBytesSize); // dispatch messages to a separate thread, but still in order for this subscription diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java index 0838b8deab7f1..79fa5c157af71 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java @@ -620,7 +620,7 @@ public final synchronized void readEntriesComplete(List entries, Object c log.debug("[{}] Distributing {} messages to {} consumers", name, entries.size(), consumerList.size()); } - long size = entries.stream().mapToLong(Entry::getLength).sum(); + long size = getTotalBytesSize(entries); updatePendingBytesToDispatch(size); // dispatch messages to a separate thread, but still in order for this subscription diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherTotalBytesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherTotalBytesTest.java new file mode 100644 index 0000000000000..239a698ab011b --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherTotalBytesTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service.persistent; + +import static org.testng.Assert.assertEquals; +import java.util.ArrayList; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.impl.EntryImpl; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class PersistentDispatcherTotalBytesTest { + + @DataProvider(name = "entryCounts") + public Object[][] entryCounts() { + return new Object[][] { + {0}, + {1}, + {32}, + {1024} + }; + } + + @Test(dataProvider = "entryCounts") + public void testGetTotalBytesSize(int entryCount) { + EntriesAndExpectedSize entries = entriesWithVaryingPayloadSizes(entryCount); + try { + assertEquals(AbstractPersistentDispatcherMultipleConsumers.getTotalBytesSize(entries.entries), + entries.expectedSize); + } finally { + entries.release(); + } + } + + private static EntriesAndExpectedSize entriesWithVaryingPayloadSizes(int entryCount) { + EntriesAndExpectedSize entries = new EntriesAndExpectedSize(entryCount); + for (int i = 0; i < entryCount; i++) { + int payloadSize = payloadSize(i); + entries.entries.add(EntryImpl.create(1, i, new byte[payloadSize])); + entries.expectedSize += payloadSize; + } + return entries; + } + + private static int payloadSize(int index) { + return index % 97 == 0 ? 4096 : 1 + ((index * 31) & 1023); + } + + private static final class EntriesAndExpectedSize { + private final ArrayList entries; + private long expectedSize; + + private EntriesAndExpectedSize(int entryCount) { + this.entries = new ArrayList<>(entryCount); + } + + private void release() { + entries.forEach(Entry::release); + } + } +} From c9151754bf1c534e361ccc2314782daccf32773f Mon Sep 17 00:00:00 2001 From: void-ptr974 Date: Tue, 30 Jun 2026 00:23:12 +0800 Subject: [PATCH 082/213] [fix][broker] Fix replicator getting stuck under rate limiter throttling and honor readBatchSize/maxReadSizeBytes on the default read path (#26005) Co-authored-by: Lari Hotari (cherry picked from commit abca10f9ce93da79a2842455dafda707042d42fa) Signed-off-by: Zixuan Liu --- .../persistent/GeoPersistentReplicator.java | 6 +- .../persistent/PersistentReplicator.java | 195 +++++-------- .../broker/service/OneWayReplicatorTest.java | 81 ++++++ .../PersistentReplicatorInflightTaskTest.java | 264 +++++++++++------- 4 files changed, 321 insertions(+), 225 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java index b4a518c51df90..1c2fa73d1082c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/GeoPersistentReplicator.java @@ -198,9 +198,9 @@ protected boolean replicateEntries(List entries, final InFlightTask inFli * Explain the result of the race-condition between: * - {@link #readMoreEntries} * - {@link #beforeTerminateOrCursorRewinding(ReasonOfWaitForCursorRewinding)} - * Since {@link #acquirePermitsIfNotFetchingSchema} and - * {@link #beforeTerminateOrCursorRewinding(ReasonOfWaitForCursorRewinding)} acquire the - * same lock, it is safe. + * Since the read scheduling path in {@link #readMoreEntries()} and + * {@link #beforeTerminateOrCursorRewinding(ReasonOfWaitForCursorRewinding)} update in-flight + * read state under the same lock, it is safe. */ beforeTerminateOrCursorRewinding(ReasonOfWaitForCursorRewinding.Fetching_Schema); inFlightTask.incCompletedEntries(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java index 9fdfd590c4547..8a8fa0bf74d59 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java @@ -38,7 +38,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; -import lombok.AllArgsConstructor; import lombok.Data; import lombok.Getter; import org.apache.bookkeeper.mledger.AsyncCallbacks; @@ -97,7 +96,7 @@ public abstract class PersistentReplicator extends AbstractReplicator resourceGroupDispatchRateLimiter = Optional.ofNullable(v); private final Object dispatchRateLimiterLock = new Object(); - private int readBatchSize; + private volatile int readBatchSize; private final int readMaxSizeBytes; private final int producerQueueThreshold; @@ -141,10 +140,8 @@ public PersistentReplicator(String localCluster, PersistentTopic localTopic, Man this.expiryMonitor = new PersistentMessageExpiryMonitor(localTopic, Codec.decode(cursor.getName()), cursor, null); - readBatchSize = Math.min( - producerQueueSize, - localTopic.getBrokerService().pulsar().getConfiguration().getDispatcherMaxReadBatchSize()); - readMaxSizeBytes = localTopic.getBrokerService().pulsar().getConfiguration().getDispatcherMaxReadSizeBytes(); + readBatchSize = getMaxReadBatchSize(); + readMaxSizeBytes = brokerService.pulsar().getConfiguration().getDispatcherMaxReadSizeBytes(); producerQueueThreshold = (int) (producerQueueSize * 0.9); this.initializeDispatchRateLimiterIfNeeded(); @@ -152,6 +149,10 @@ public PersistentReplicator(String localCluster, PersistentTopic localTopic, Man startProducer(); } + private int getMaxReadBatchSize() { + return Math.min(producerQueueSize, brokerService.pulsar().getConfiguration().getDispatcherMaxReadBatchSize()); + } + @Override protected void setProducerAndTriggerReadEntries(Producer producer) { /** @@ -225,61 +226,48 @@ protected void disableReplicatorRead() { this.cursor.setInactive(); } - @Data - @AllArgsConstructor - private static class AvailablePermits { - private int messages; - private long bytes; - - /** - * messages, bytes - * 0, O: Producer queue is full, no permits. - * -1, -1: Rate Limiter reaches limit. - * >0, >0: available permits for read entries. - */ - public boolean isExceeded() { - return messages == -1 && bytes == -1; - } - + private record ReadLimits(int messages, long bytes) { public boolean isReadable() { return messages > 0 && bytes > 0; } } /** - * Calculate available permits for read entries. + * Calculate read limits for a read operation. Takes the rate limiter into account if it's enabled. + * Also limits to current readBatchSize and readMaxSizeBytes. */ - private AvailablePermits getRateLimiterAvailablePermits(int availablePermits) { + private ReadLimits getReadLimits(int permits) { // return 0, if Producer queue is full, it will pause read entries. - if (availablePermits <= 0) { + if (permits <= 0) { if (log.isDebugEnabled()) { - log.debug("[{}] Producer queue is full, availablePermits: {}, pause reading", - replicatorId, availablePermits); + log.debug("[{}] Producer queue is full, permits: {}, pause reading", replicatorId, permits); } - return new AvailablePermits(0, 0); + return new ReadLimits(0, 0); } - long availablePermitsOnMsg = -1; - long availablePermitsOnByte = -1; + long readLimitOnMsg = -1; + long readLimitOnByte = -1; // handle rate limit if (dispatchRateLimiter.isPresent() && dispatchRateLimiter.get().isDispatchRateLimitingEnabled()) { DispatchRateLimiter rateLimiter = dispatchRateLimiter.get(); - // if dispatch-rate is in msg then read only msg according to available permit - availablePermitsOnMsg = rateLimiter.getAvailableDispatchRateLimitOnMsg(); - availablePermitsOnByte = rateLimiter.getAvailableDispatchRateLimitOnByte(); - // no permits from rate limit - if (availablePermitsOnByte == 0 || availablePermitsOnMsg == 0) { + // rateLimiter returns -1 if there is no rate limit configured + readLimitOnMsg = rateLimiter.getAvailableDispatchRateLimitOnMsg(); + readLimitOnByte = rateLimiter.getAvailableDispatchRateLimitOnByte(); + // no permits from rate limit when either limit is 0 + if (readLimitOnByte == 0 || readLimitOnMsg == 0) { if (log.isDebugEnabled()) { - log.debug("[{}] message-read exceeded topic replicator message-rate {}/{}," - + " schedule after a {}", + log.debug("[{}] Message-read exceeded topic replicator rate limit," + + " dispatchRateOnMsg: {}, dispatchRateOnByte: {}," + + " readLimitOnMsg: {}, readLimitOnByte: {}", replicatorId, rateLimiter.getDispatchRateOnMsg(), rateLimiter.getDispatchRateOnByte(), - MESSAGE_RATE_BACKOFF_MS); + readLimitOnMsg, + readLimitOnByte); } - return new AvailablePermits(-1, -1); + return new ReadLimits(-1, -1); } } @@ -296,28 +284,21 @@ private AvailablePermits getRateLimiterAvailablePermits(int availablePermits) { rateLimiter.getDispatchRateOnByte(), MESSAGE_RATE_BACKOFF_MS); } - return new AvailablePermits(-1, -1); - } - if (availablePermitsOnMsg == -1) { - availablePermitsOnMsg = rgAvailablePermitsOnMsg; - } else { - availablePermitsOnMsg = Math.min(rgAvailablePermitsOnMsg, availablePermitsOnMsg); - } - if (availablePermitsOnByte == -1) { - availablePermitsOnByte = rgAvailablePermitsOnByte; - } else { - availablePermitsOnByte = Math.min(rgAvailablePermitsOnByte, availablePermitsOnByte); + return new ReadLimits(-1, -1); } + readLimitOnMsg = + readLimitOnMsg == -1 ? rgAvailablePermitsOnMsg : Math.min(readLimitOnMsg, rgAvailablePermitsOnMsg); + readLimitOnByte = readLimitOnByte == -1 ? rgAvailablePermitsOnByte : + Math.min(readLimitOnByte, rgAvailablePermitsOnByte); } - availablePermitsOnMsg = - availablePermitsOnMsg == -1 ? availablePermits : Math.min(availablePermits, availablePermitsOnMsg); - availablePermitsOnMsg = Math.min(availablePermitsOnMsg, readBatchSize); + readLimitOnMsg = readLimitOnMsg == -1 ? permits : Math.min(permits, readLimitOnMsg); - availablePermitsOnByte = - availablePermitsOnByte == -1 ? readMaxSizeBytes : Math.min(readMaxSizeBytes, availablePermitsOnByte); + readLimitOnByte = + readLimitOnByte == -1 ? readMaxSizeBytes : Math.min(readMaxSizeBytes, readLimitOnByte); - return new AvailablePermits((int) availablePermitsOnMsg, availablePermitsOnByte); + + return new ReadLimits((int) readLimitOnMsg, readLimitOnByte); } public void disconnectIfNoTrafficAndBacklog() { @@ -348,8 +329,39 @@ protected void readMoreEntries() { if (state.equals(Terminated) || state.equals(Terminating)) { return; } - // Acquire permits and check state of producer. - InFlightTask newInFlightTask = acquirePermitsIfNotFetchingSchema(); + InFlightTask newInFlightTask = null; + ReadLimits readLimits = null; + synchronized (inFlightTasks) { + if (hasPendingRead()) { + log.debug("Skip the reading because there is a pending read task"); + } else if (waitForCursorRewindingRefCnf > 0) { + log.debug("Skip the reading due to new detected schema"); + } else if (state != Started) { + log.debug("Skip the reading because producer has not started"); + } else { + int permits = getPermitsIfNoPendingRead(); + if (permits > 0) { + if (!isWritable()) { + log.debug("Throttling replication traffic to a single message permit because producer is not " + + "writable"); + // Minimize the read size if the producer is disconnected or the window is already full. + permits = 1; + } + + readLimits = getReadLimits(permits); + if (readLimits.isReadable()) { + newInFlightTask = createOrRecycleInFlightTaskIntoQueue(cursor.getReadPosition(), + readLimits.messages); + } else { + // no rate limiter permits from rate limit + if (log.isDebugEnabled()) { + log.debug("[{}] Throttling replication traffic. Messages To Read {}, Bytes To Read {}", + replicatorId, readLimits.messages, readLimits.bytes); + } + } + } + } + } if (newInFlightTask == null) { // no permits from rate limit if (log.isDebugEnabled()) { @@ -358,48 +370,14 @@ protected void readMoreEntries() { if (!hasPendingRead()) { topic.getBrokerService().executor().schedule( () -> readMoreEntries(), MESSAGE_RATE_BACKOFF_MS, TimeUnit.MILLISECONDS); - return; - } else { - return; - } - } - // If disabled RateLimiter. - if (!dispatchRateLimiter.isPresent() || !dispatchRateLimiter.get().isDispatchRateLimitingEnabled()) { - cursor.asyncReadEntriesOrWait(newInFlightTask.readingEntries, -1, this, - newInFlightTask/* Context object */, topic.getMaxReadPosition()); - return; - } - // No permits of RateLimiter. - AvailablePermits availablePermits = getRateLimiterAvailablePermits(newInFlightTask.readingEntries); - if (!availablePermits.isReadable()) { - // no rate limiter permits from rate limit - if (log.isDebugEnabled()) { - log.debug("[{}] Throttling replication traffic. Messages To Read {}, Bytes To Read {}", - replicatorId, availablePermits.getMessages(), availablePermits.getBytes()); } - topic.getBrokerService().executor().schedule( - () -> readMoreEntries(), MESSAGE_RATE_BACKOFF_MS, TimeUnit.MILLISECONDS); return; } - // Has permits of RateLimiter. - int messagesToRead = availablePermits.getMessages(); - long bytesToRead = availablePermits.getBytes(); - if (!isWritable()) { - if (log.isDebugEnabled()) { - log.debug("[{}] Throttling replication traffic because producer is not writable", replicatorId); - } - // Minimize the read size if the producer is disconnected or the window is already full - messagesToRead = 1; - } - // Update acquired permits exceeds limitation. - if (messagesToRead < newInFlightTask.readingEntries) { - newInFlightTask.setReadingEntries(messagesToRead); - } if (log.isDebugEnabled()) { - log.debug("[{}] Schedule read of {} messages or {} bytes", replicatorId, newInFlightTask.readingEntries, - bytesToRead); + log.debug("[{}] Schedule read of {} messages or {} bytes", replicatorId, + newInFlightTask.readingEntries, readLimits.bytes); } - cursor.asyncReadEntriesOrWait(newInFlightTask.readingEntries, bytesToRead, this, + cursor.asyncReadEntriesOrWait(newInFlightTask.readingEntries, readLimits.bytes, this, newInFlightTask/* Context object */, topic.getMaxReadPosition()); } @@ -454,7 +432,7 @@ public void readEntriesComplete(List entries, Object ctx) { inFlightTask.setEntries(entries); // After the replicator starts, the speed will be gradually increased. - int maxReadBatchSize = topic.getBrokerService().pulsar().getConfiguration().getDispatcherMaxReadBatchSize(); + int maxReadBatchSize = getMaxReadBatchSize(); if (readBatchSize < maxReadBatchSize) { int newReadBatchSize = Math.min(readBatchSize * 2, maxReadBatchSize); if (log.isDebugEnabled()) { @@ -613,7 +591,7 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { } // Reduce read batch size to avoid flooding bookies with retries - readBatchSize = topic.getBrokerService().pulsar().getConfiguration().getDispatcherMinReadBatchSize(); + readBatchSize = brokerService.pulsar().getConfiguration().getDispatcherMinReadBatchSize(); long waitTimeMillis = readFailureBackoff.next(); @@ -1017,29 +995,6 @@ InFlightTask createOrRecycleInFlightTaskIntoQueue(Position readPos, int readingE } } - protected InFlightTask acquirePermitsIfNotFetchingSchema() { - synchronized (inFlightTasks) { - if (hasPendingRead()) { - log.info("[{}] Skip the reading because there is a pending read task", replicatorId); - return null; - } - if (waitForCursorRewindingRefCnf > 0) { - log.info("[{}] Skip the reading due to new detected schema", replicatorId); - return null; - } - if (state != Started) { - log.info("[{}] Skip the reading because producer has not started [{}]", replicatorId, state); - return null; - } - // Guarantee that there is a unique cursor reading task. - int permits = getPermitsIfNoPendingRead(); - if (permits == 0) { - return null; - } - return createOrRecycleInFlightTaskIntoQueue(cursor.getReadPosition(), permits); - } - } - protected int getPermitsIfNoPendingRead() { synchronized (inFlightTasks) { for (InFlightTask task : inFlightTasks) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java index df1d18a00dac1..5e6a78305da8d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java @@ -95,6 +95,7 @@ import org.apache.pulsar.client.api.ProducerBuilder; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.impl.ClientBuilderImpl; import org.apache.pulsar.client.impl.ClientCnx; @@ -108,6 +109,7 @@ import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride; import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.DispatchRate; import org.apache.pulsar.common.policies.data.HierarchyTopicPolicies; import org.apache.pulsar.common.policies.data.PublishRate; import org.apache.pulsar.common.policies.data.ReplicatorStats; @@ -2204,4 +2206,83 @@ public void testReplicatorsInflightTaskListIsEmptyAfterReplicationFinished() thr // Verify: all inflight tasks are done. ensureNoBacklogByInflightTask(getReplicator(topicName)); } + + @DataProvider + public Object[][] replicatorDispatchRateLimits() { + return new Object[][] { + {1, -1L}, + {-1, 1L} + }; + } + + @Test(timeOut = 90_000, dataProvider = "replicatorDispatchRateLimits") + public void testReplicatorContinuesAfterRateLimiterHasNoPermits(int messageRate, long byteRate) throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + replicatedNamespace + "/tp_"); + final String subscriptionName = "sub"; + final List messages = Arrays.asList("msg-0", "msg-1", "msg-2"); + DispatchRate dispatchRate = DispatchRate.builder() + .dispatchThrottlingRateInMsg(messageRate) + .dispatchThrottlingRateInByte(byteRate) + .ratePeriodInSecond(2) + .build(); + Producer producer = null; + Consumer consumer = null; + boolean topicCreated = false; + boolean dispatchRateConfigured = false; + try { + admin1.topics().createNonPartitionedTopic(topicName); + topicCreated = true; + admin1.topicPolicies().setReplicatorDispatchRate(topicName, dispatchRate); + dispatchRateConfigured = true; + GeoPersistentReplicator replicator = getReplicator(topicName); + Awaitility.await().untilAsserted(() -> { + assertTrue(replicator.getRateLimiter().isPresent()); + assertEquals(replicator.getRateLimiter().get().getDispatchRateOnMsg(), messageRate); + assertEquals(replicator.getRateLimiter().get().getDispatchRateOnByte(), byteRate); + }); + consumer = client2.newConsumer(Schema.STRING) + .topic(topicName) + .subscriptionName(subscriptionName) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + producer = client1.newProducer(Schema.STRING) + .topic(topicName) + .enableBatching(false) + .create(); + + for (String message : messages) { + producer.send(message); + } + + Set expected = new HashSet<>(messages); + Set received = new HashSet<>(); + Consumer subscribedConsumer = consumer; + Awaitility.await().atMost(Duration.ofSeconds(60)).untilAsserted(() -> { + Message message = subscribedConsumer.receive(1, TimeUnit.SECONDS); + if (message != null) { + received.add(message.getValue()); + subscribedConsumer.acknowledge(message); + } + assertEquals(received, expected); + }); + waitForReplicationTaskFinish(topicName); + ensureNoBacklogByInflightTask(replicator); + } finally { + if (producer != null) { + producer.close(); + } + if (consumer != null) { + consumer.close(); + } + if (dispatchRateConfigured) { + admin1.topicPolicies().removeReplicatorDispatchRate(topicName); + } + if (topicCreated) { + admin1.topics().setReplicationClusters(topicName, Arrays.asList(cluster1)); + waitReplicatorStopped(topicName, false); + admin1.topics().delete(topicName, true); + admin2.topics().delete(topicName, true); + } + } + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java index 83529969df417..a34e86fd11837 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java @@ -18,44 +18,64 @@ */ package org.apache.pulsar.broker.service.persistent; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import io.netty.channel.EventLoopGroup; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.LinkedList; import java.util.List; +import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerTest; import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.AbstractReplicator; -import org.apache.pulsar.broker.service.BrokerServiceInternalMethodInvoker; +import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.OneWayReplicatorTestBase; import org.apache.pulsar.broker.service.persistent.PersistentReplicator.InFlightTask; import org.apache.pulsar.broker.service.persistent.PersistentReplicator.ProducerSendCallback; import org.apache.pulsar.broker.service.persistent.PersistentReplicator.ReasonOfWaitForCursorRewinding; +import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerBuilder; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.PulsarClientImpl; import org.awaitility.Awaitility; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @Slf4j @@ -199,6 +219,63 @@ public void testFailedPublishCompletesInFlightTask() throws Exception { } } + @DataProvider + public Object[][] readSchedulingLimits() { + return new Object[][] { + {"message permits exhausted", 0, -1L, true, false, 0, 0L}, + {"byte permits exhausted", -1, 0L, true, false, 0, 0L}, + {"message permits limit read batch", 5, -1L, true, true, 5, 1024L}, + {"byte permits limit read size", -1, 512L, true, true, 100, 512L}, + {"non-writable producer limits read batch", 5, 512L, false, true, 1, 512L} + }; + } + + @Test(dataProvider = "readSchedulingLimits") + public void testReadMoreEntriesSchedulesCursorReadWithReadLimits(String scenario, + long availableMessages, + long availableBytes, + boolean writable, + boolean expectRead, + int expectedMessages, + long expectedBytes) throws Exception { + TestReplicatorFixture fixture = newTestReplicatorFixture(writable); + PersistentReplicator replicator = fixture.replicator; + DispatchRateLimiter rateLimiter = mock(DispatchRateLimiter.class); + when(rateLimiter.isDispatchRateLimitingEnabled()).thenReturn(true); + when(rateLimiter.getAvailableDispatchRateLimitOnMsg()).thenReturn(availableMessages); + when(rateLimiter.getAvailableDispatchRateLimitOnByte()).thenReturn(availableBytes); + replicator.dispatchRateLimiter = Optional.of(rateLimiter); + + replicator.readMoreEntries(); + + if (expectRead) { + assertEquals(replicator.inFlightTasks.size(), 1, scenario); + InFlightTask inFlightTask = replicator.inFlightTasks.peek(); + verify(fixture.cursor).asyncReadEntriesOrWait(eq(expectedMessages), eq(expectedBytes), + same(replicator), same(inFlightTask), any(Position.class)); + assertEquals(inFlightTask.getReadingEntries(), expectedMessages, scenario); + verify(fixture.executor, never()).schedule(any(Runnable.class), anyLong(), any(TimeUnit.class)); + } else { + verify(fixture.cursor, never()).asyncReadEntriesOrWait(anyInt(), anyLong(), any(), any(), any()); + assertTrue(replicator.inFlightTasks.isEmpty(), scenario); + verify(fixture.executor).schedule(any(Runnable.class), eq((long) PersistentTopic.MESSAGE_RATE_BACKOFF_MS), + eq(TimeUnit.MILLISECONDS)); + } + } + + @Test + public void testReadMoreEntriesSkipsReadWhenPendingReadExists() throws Exception { + TestReplicatorFixture fixture = newTestReplicatorFixture(true); + PersistentReplicator replicator = fixture.replicator; + replicator.inFlightTasks.add(new InFlightTask(PositionFactory.create(1, 1), 5, replicator.getReplicatorId())); + + replicator.readMoreEntries(); + + verify(fixture.cursor, never()).asyncReadEntriesOrWait(anyInt(), anyLong(), any(), any(), any()); + verify(fixture.executor, never()).schedule(any(Runnable.class), anyLong(), any(TimeUnit.class)); + assertEquals(replicator.inFlightTasks.size(), 1); + } + @Test public void testCreateOrRecycleInFlightTaskIntoQueue() throws Exception { log.info("Starting testCreateOrRecycleInFlightTaskIntoQueue"); @@ -389,111 +466,93 @@ public void testGetPermitsIfNoPendingRead() throws Exception { } } - @Test - public void testAcquirePermitsIfNotFetchingSchema() throws Exception { - log.info("Starting testAcquirePermitsIfNotFetchingSchema"); - // Get the replicator for the test topic - PersistentReplicator replicator = getReplicator(topicName); - Assert.assertNotNull(replicator, "Replicator should not be null"); + @SuppressWarnings("unchecked") + private TestReplicatorFixture newTestReplicatorFixture(boolean writable) throws Exception { + ServiceConfiguration configuration = new ServiceConfiguration(); + configuration.setClusterName("local"); + configuration.setReplicationProducerQueueSize(1000); + configuration.setDispatcherMaxReadBatchSize(100); + configuration.setDispatcherMaxReadSizeBytes(1024); + + PulsarService pulsar = mock(PulsarService.class); + when(pulsar.getConfiguration()).thenReturn(configuration); + when(pulsar.getConfig()).thenReturn(configuration); + when(pulsar.getClient()).thenReturn(mock(PulsarClientImpl.class)); + when(pulsar.getAdminClient()).thenReturn(mock(PulsarAdmin.class)); + + BrokerService brokerService = mock(BrokerService.class); + EventLoopGroup executor = mock(EventLoopGroup.class); + when(brokerService.pulsar()).thenReturn(pulsar); + when(brokerService.getPulsar()).thenReturn(pulsar); + when(brokerService.executor()).thenReturn(executor); + + ProducerBuilder producerBuilder = mock(ProducerBuilder.class); + when(producerBuilder.topic(anyString())).thenReturn(producerBuilder); + when(producerBuilder.messageRoutingMode(any())).thenReturn(producerBuilder); + when(producerBuilder.enableBatching(anyBoolean())).thenReturn(producerBuilder); + when(producerBuilder.sendTimeout(anyInt(), any(TimeUnit.class))).thenReturn(producerBuilder); + when(producerBuilder.maxPendingMessages(anyInt())).thenReturn(producerBuilder); + when(producerBuilder.producerName(anyString())).thenReturn(producerBuilder); + + PulsarClientImpl replicationClient = mock(PulsarClientImpl.class); + when(replicationClient.newProducer(any(Schema.class))).thenReturn(producerBuilder); + + PersistentTopic topic = mock(PersistentTopic.class); + when(topic.getName()).thenReturn("persistent://prop/ns/test-read-scheduling"); + when(topic.getReplicatorPrefix()).thenReturn("pulsar.repl"); + when(topic.getBrokerService()).thenReturn(brokerService); + when(topic.getMaxReadPosition()).thenReturn(PositionFactory.create(1, 100)); + + ManagedCursor cursor = mock(ManagedCursor.class); + when(cursor.getName()).thenReturn("pulsar.repl.remote"); + when(cursor.getReadPosition()).thenReturn(PositionFactory.create(1, 1)); + + TestPersistentReplicator replicator = new TestPersistentReplicator(topic, cursor, brokerService, + replicationClient, writable); + return new TestReplicatorFixture(replicator, cursor, executor); + } - // Get access to the inFlightTasks list for setup - LinkedList inFlightTasks = replicator.inFlightTasks; - Assert.assertNotNull(inFlightTasks, "InFlightTasks list should not be null"); + private static class TestReplicatorFixture { + final TestPersistentReplicator replicator; + final ManagedCursor cursor; + final EventLoopGroup executor; - // Save original tasks and clear for testing - List originalTasks = new ArrayList<>(inFlightTasks); - inFlightTasks.clear(); + TestReplicatorFixture(TestPersistentReplicator replicator, ManagedCursor cursor, EventLoopGroup executor) { + this.replicator = replicator; + this.cursor = cursor; + this.executor = executor; + } + } - // Save original state - int originalWaitForCursorRewinding = replicator.waitForCursorRewindingRefCnf; - AbstractReplicator.State originalState = replicator.getState(); + private static class TestPersistentReplicator extends PersistentReplicator { + private final boolean writable; - try { - // Test Case 1: Normal case - no pending read, not waiting for cursor rewinding, state is Started - // Should return a new InFlightTask - // First, check the current permits available - int expectedPermits = replicator.getPermitsIfNoPendingRead(); - Assert.assertTrue(expectedPermits > 0, "Should have available permits for the test"); - InFlightTask task1 = replicator.acquirePermitsIfNotFetchingSchema(); - Assert.assertNotNull(task1, "Should return a new InFlightTask in normal case"); - Assert.assertNotNull(task1.getReadPos(), "Task should have a read position"); - Assert.assertEquals(task1.getReadingEntries(), expectedPermits, - "Task readingEntries should equal the number of permits available"); - Assert.assertTrue(inFlightTasks.contains(task1), - "Task should be added to the inFlightTasks list"); - - // Test Case 2: With pending read - should return null - inFlightTasks.clear(); - Position position1 = PositionFactory.create(1, 1); - InFlightTask pendingReadTask = new InFlightTask(position1, 5, ""); - // Don't set readoutEntries to simulate pending read - inFlightTasks.add(pendingReadTask); - InFlightTask task2 = replicator.acquirePermitsIfNotFetchingSchema(); - Assert.assertNull(task2, "Should return null when there is a pending read"); + TestPersistentReplicator(PersistentTopic topic, ManagedCursor cursor, BrokerService brokerService, + PulsarClientImpl replicationClient, boolean writable) + throws PulsarServerException { + super("local", topic, cursor, "remote", topic.getName(), brokerService, replicationClient); + this.writable = writable; + this.state = State.Started; + } - // Test Case 3: With waitForCursorRewinding=true - should return null - inFlightTasks.clear(); - replicator.waitForCursorRewindingRefCnf = 1; - InFlightTask task3 = replicator.acquirePermitsIfNotFetchingSchema(); - Assert.assertNull(task3, "Should return null when waiting for cursor rewinding"); - // Reset for next test - replicator.waitForCursorRewindingRefCnf = 0; - - // Test Case 4: With state != Started - should return null - // We need to use reflection to modify the state since it's protected by AtomicReferenceFieldUpdater - BrokerServiceInternalMethodInvoker.replicatorSetState(replicator, AbstractReplicator.State.Starting); - InFlightTask task4 = replicator.acquirePermitsIfNotFetchingSchema(); - Assert.assertNull(task4, "Should return null when state is not Started"); - // Reset state for next test - BrokerServiceInternalMethodInvoker.replicatorSetState(replicator, AbstractReplicator.State.Started); - - // Test Case 5: With limited permits - verify readingEntries is set correctly - inFlightTasks.clear(); - // Add a task with some in-flight messages to reduce available permits - Position positionLimited = PositionFactory.create(10, 10); - InFlightTask limitedTask = new InFlightTask(positionLimited, 5, ""); - // Add enough entries to leave just a small number of permits (e.g., 10) - List limitedEntries = new ArrayList<>(); - int entriesCount = 990; - for (int j = 0; j < entriesCount; j++) { - limitedEntries.add(mock(Entry.class)); - } - limitedTask.setEntries(limitedEntries); - inFlightTasks.add(limitedTask); - // Check that we have limited permits available - int limitedPermits = replicator.getPermitsIfNoPendingRead(); - Assert.assertTrue(limitedPermits > 0 && limitedPermits < 20, - "Should have a small number of permits available for testing"); - // Now acquire permits and verify readingEntries matches the limited permits - InFlightTask task5 = replicator.acquirePermitsIfNotFetchingSchema(); - Assert.assertNotNull(task5, "Should return a task with limited permits"); - Assert.assertEquals(task5.getReadingEntries(), limitedPermits, - "Task readingEntries should equal the limited number of permits available"); - - // Test Case 6: With permits=0 - should return null - inFlightTasks.clear(); - // Add tasks that will make getPermitsIfNoPendingRead() return 0 - // We need enough in-flight messages to equal producerQueueSize - for (int i = 0; i < 10; i++) { - Position position = PositionFactory.create(i, i); - InFlightTask task = new InFlightTask(position, 5, ""); - List entries = new ArrayList<>(); - for (int j = 0; j < 100; j++) { - entries.add(mock(Entry.class)); - } - task.setEntries(entries); - inFlightTasks.add(task); - } - InFlightTask task6 = replicator.acquirePermitsIfNotFetchingSchema(); - Assert.assertNull(task6, "Should return null when permits is 0"); - log.info("Completed testAcquirePermitsIfNotFetchingSchema"); - } finally { - // Restore original state - replicator.waitForCursorRewindingRefCnf = originalWaitForCursorRewinding; - BrokerServiceInternalMethodInvoker.replicatorSetState(replicator, originalState); - // Restore original tasks - inFlightTasks.clear(); - inFlightTasks.addAll(originalTasks); + @Override + protected void startProducer() { + // No-op for scheduling behavior tests. + } + + @Override + protected String getProducerName() { + return "test-replicator"; + } + + @Override + protected boolean isWritable() { + return writable; + } + + @Override + protected boolean replicateEntries(List entries, InFlightTask inFlightTask) { + return true; } } @@ -503,7 +562,8 @@ public static Runnable pauseReplicator(PersistentReplicator replicator) { }); replicator.beforeTerminateOrCursorRewinding(PersistentReplicator.ReasonOfWaitForCursorRewinding.Disconnecting); replicator.doRewindCursor(false); - InFlightTask inFlightTask = replicator.createOrRecycleInFlightTaskIntoQueue(PositionFactory.create(1, 1), 1); + InFlightTask inFlightTask = + replicator.createOrRecycleInFlightTaskIntoQueue(PositionFactory.create(1, 1), 1); return () -> { inFlightTask.setEntries(Collections.emptyList()); replicator.readMoreEntries(); From 5cdb612d3402116a3095eed091ef44e832d057e2 Mon Sep 17 00:00:00 2001 From: sinan liu Date: Tue, 30 Jun 2026 00:58:18 +0800 Subject: [PATCH 083/213] [fix][broker] Forward topic policy updates after init failures (#26110) (cherry picked from commit b14524eb672181124f8834a2048b8ed6a278385d) --- .../service/persistent/PersistentTopic.java | 22 ++++++--- .../persistent/PersistentTopicTest.java | 46 +++++++++++++++++++ 2 files changed, 62 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 388de5ceb8390..afb5933789a66 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -4747,12 +4747,22 @@ protected CompletableFuture initTopicPolicy() { CompletableFuture> localPoliciesFuture = topicPoliciesService.getTopicPoliciesAsync(partitionedTopicName, TopicPoliciesService.GetType.LOCAL_ONLY); - return globalPoliciesFuture.thenCombine(localPoliciesFuture, (global, local) -> { - // finally update the topic policies with the latest value or loaded value - return CompletableFuture.runAsync(() -> - topicPolicyListener.completeInitialization(global.orElse(null), local.orElse(null)), - getPoliciesNotifyThread()); - }).thenCompose(Function.identity()); + CompletableFuture initialPoliciesFuture = + globalPoliciesFuture.thenCombine(localPoliciesFuture, (global, local) -> { + // finally update the topic policies with the latest value or loaded value + return CompletableFuture.runAsync(() -> + topicPolicyListener.completeInitialization(global.orElse(null), + local.orElse(null)), + getPoliciesNotifyThread()); + }).thenCompose(Function.identity()); + return initialPoliciesFuture.exceptionallyCompose(ex -> + // The topic load path logs and continues when initial policy loading fails. Make sure the + // already-registered wrapper is not left buffering future live updates forever. + CompletableFuture.runAsync( + () -> topicPolicyListener.completeInitialization(null, null), + getPoliciesNotifyThread()) + .thenCompose(__ -> FutureUtil.failedFuture( + FutureUtil.unwrapCompletionException(ex)))); }); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java index 364360573da70..7717595b2732b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java @@ -64,6 +64,7 @@ import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.ManagedCursorContainer; @@ -74,6 +75,7 @@ import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.TopicPoliciesService; +import org.apache.pulsar.broker.service.TopicPolicyListener; import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.Metric; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; @@ -97,6 +99,7 @@ import org.apache.pulsar.common.policies.data.TopicPolicies; import org.apache.pulsar.common.policies.data.TopicStats; import org.awaitility.Awaitility; +import org.mockito.ArgumentCaptor; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -727,6 +730,49 @@ public void testCheckPersistencePolicies() throws Exception { TimeUnit.MINUTES.toMillis(1)); } + @Test + public void testTopicPolicyListenerForwardsLiveUpdatesAfterInitialLoadFailure() throws Exception { + class RecordingPersistentTopic extends PersistentTopic { + final List receivedUpdates = new ArrayList<>(); + + RecordingPersistentTopic(String topic, ManagedLedger ledger, BrokerService brokerService) { + super(topic, ledger, brokerService); + } + + @Override + public void onUpdate(TopicPolicies policies) { + receivedUpdates.add(policies); + } + } + + final String topic = "persistent://prop/ns-abc/testTopicPolicyInitFailure-" + UUID.randomUUID(); + ManagedLedger ledger = mock(ManagedLedger.class); + doReturn(new ManagedLedgerConfig()).when(ledger).getConfig(); + doReturn(Collections.emptyMap()).when(ledger).getProperties(); + + TopicPoliciesService policiesService = mock(TopicPoliciesService.class); + doReturn(policiesService).when(pulsar).getTopicPoliciesService(); + doReturn(CompletableFuture.completedFuture(true)).when(policiesService) + .registerListenerAsync(any(TopicName.class), any(TopicPolicyListener.class)); + doReturn(CompletableFuture.failedFuture(new RuntimeException("initial topic policy load failed"))) + .when(policiesService).getTopicPoliciesAsync(any(TopicName.class), + any(TopicPoliciesService.GetType.class)); + + RecordingPersistentTopic persistentTopic = + new RecordingPersistentTopic(topic, ledger, pulsar.getBrokerService()); + persistentTopic.initTopicPolicy().handle((ignored, ex) -> null).get(3, TimeUnit.SECONDS); + + ArgumentCaptor listenerCaptor = ArgumentCaptor.forClass(TopicPolicyListener.class); + verify(policiesService).registerListenerAsync(any(TopicName.class), listenerCaptor.capture()); + + TopicPolicies livePolicies = new TopicPolicies(); + livePolicies.setIsGlobal(false); + livePolicies.setMaxConsumerPerTopic(10); + listenerCaptor.getValue().onUpdate(livePolicies); + + assertEquals(persistentTopic.receivedUpdates, Collections.singletonList(livePolicies)); + } + @Test public void testDynamicConfigurationAutoSkipNonRecoverableData() throws Exception { pulsar.getConfiguration().setAutoSkipNonRecoverableData(false); From 1fa30987830519c7929aaa0ba3fbd9ba6bdc35e1 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 29 Jun 2026 21:46:27 +0300 Subject: [PATCH 084/213] [fix][test][branch-4.0] Adapt ConfigurationDataUtilsTest to Jackson 2.18.8 InetSocketAddress deserialization --- .../client/impl/conf/ConfigurationDataUtilsTest.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ConfigurationDataUtilsTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ConfigurationDataUtilsTest.java index a69e58b2eee48..00679aa2d0a1d 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ConfigurationDataUtilsTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ConfigurationDataUtilsTest.java @@ -77,7 +77,15 @@ public void testLoadClientConfigurationData() { assertEquals("v2", confData.getAuthParamMap().get("k2")); assertEquals("0.0.0.0", confData.getDnsLookupBindAddress()); assertEquals(0, confData.getDnsLookupBindPort()); - assertEquals(dnsServerAddresses, confData.getDnsServerAddresses()); + // jackson-databind 2.18.8+/2.22+ defers DNS resolution when deserializing InetSocketAddress + // (CVE-2026-54514 fix), which changes the resolved/unresolved representation. Compare host + // and port — the values that must survive the config round-trip — instead of object equality. + List loadedDnsServerAddresses = confData.getDnsServerAddresses(); + assertEquals(loadedDnsServerAddresses.size(), dnsServerAddresses.size()); + for (int i = 0; i < dnsServerAddresses.size(); i++) { + assertEquals(loadedDnsServerAddresses.get(i).getHostString(), dnsServerAddresses.get(i).getHostString()); + assertEquals(loadedDnsServerAddresses.get(i).getPort(), dnsServerAddresses.get(i).getPort()); + } } @Test From d92f4c409dd0efddb4bbf504ab3a2c8483d763a3 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 30 Jun 2026 01:40:50 +0300 Subject: [PATCH 085/213] Revert "[improve][test]Add test: test/testTopicPartitionCannotBeCreatedAfterTopicDeleted (#26038)" This reverts commit 85ee3b9bc9cdea1d56ddba016ff896f21bd33301. The test testTopicPartitionCannotBeCreatedAfterTopicDeleted (added by #26038) is a regression test for the topic consistency check introduced on master by #24118 ("[fix][broker] Add topic consistency check"). That broker-side check (NamespaceService/BrokerService) prevents a partition from being re-created / loaded once its partitioned-topic metadata has been deleted, which is what the test asserts via assertFalse(producer.isConnected()). On branch-4.0, #24118 was deliberately reverted (eef20ed340c) and its master replacement PIP-414 (#24213) was never backported, so the consistency-check behavior the test relies on is absent. Without it, a reconnecting producer can race the topic deletion and re-create the partition, so the producer stays connected and the test fails ("expected [false] but found [true]", SimpleProducerConsumerTest.java:5465). #26038 is a test-only PR that should not have been cherry-picked to branch-4.0 without its prerequisite production fix. Reverting the test restores a green build; the underlying feature remains intentionally absent on this branch. --- .../api/SimpleProducerConsumerTest.java | 60 ------------------- 1 file changed, 60 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java index 8b56d593b4779..c553950014f6e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java @@ -110,7 +110,6 @@ import org.apache.pulsar.client.impl.PartitionedProducerImpl; import org.apache.pulsar.client.impl.ProducerBase; import org.apache.pulsar.client.impl.ProducerImpl; -import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.TopicMessageImpl; import org.apache.pulsar.client.impl.TypedMessageBuilderImpl; import org.apache.pulsar.client.impl.crypto.MessageCryptoBc; @@ -5411,65 +5410,6 @@ public void testBacklogAfterCreatedSubscription(boolean trimLegderBeforeGetStats admin.topics().delete(topic, false); } - @Test - public void testTopicPartitionCannotBeCreatedAfterTopicDeleted() throws Exception { - final String topic = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); - admin.topics().createPartitionedTopic(topic, 1); - - // Inject an delay: delay to handle channel inactive event, to let the producer delay to reconnect. - ClientBuilderImpl clientBuilder = (ClientBuilderImpl) PulsarClient.builder() - .serviceUrl(lookupUrl.toString()) - .statsInterval(0, TimeUnit.SECONDS) - .connectionsPerBroker(1); - CountDownLatch countDownLatch = new CountDownLatch(1); - PulsarClientImpl pulsarClient = InjectedClientCnxClientBuilder.create(clientBuilder, (conf, eventLoopGroup) -> { - - return new ClientCnx(InstrumentProvider.NOOP, conf, eventLoopGroup) { - - @Override - public void channelInactive(ChannelHandlerContext ctx) throws Exception { - // Delay receiving the event, let producer will not reconnect immediately. - log.info("channel inactive"); - countDownLatch.await(); - super.channelInactive(ctx); - } - }; - }); - - // Producer connected. - Producer producer = pulsarClient.newProducer(Schema.BYTES) - .topic(topic) - .create(); - PersistentTopic persistentTopic1 = (PersistentTopic) pulsar.getBrokerService() - .getTopic(topic + "-partition-0", false) - .get(5, TimeUnit.SECONDS).get(); - Awaitility.await().untilAsserted(() -> { - Assert.assertEquals(persistentTopic1.getProducers().values().size(), 1); - }); - - // Make a network issue which leads to the connection breaks. - org.apache.pulsar.broker.service.Producer serviceProducer = persistentTopic1.getProducers().values() - .iterator().next(); - ServerCnx servercnx = (ServerCnx) serviceProducer.getCnx(); - servercnx.ctx().close(); - // After the connection is break, and before the producer reconnects, the partitioned topic can be deleted - // without "--force". - admin.topics().deletePartitionedTopic(topic); - Awaitility.await().untilAsserted(() -> { - assertTrue(persistentTopic1.isClosingOrDeleting()); - }); - - // Verify: the partition can not be loaded up once the partitioned topic was deleted. - countDownLatch.countDown(); - Thread.sleep(10_000); - assertFalse(producer.isConnected()); - assertFalse(pulsar.getBrokerService().getTopics().containsKey(topic + "-partition-0")); - - // cleanup. - producer.close(); - pulsarClient.close(); - } - /** * The internal producer of replicator will resend messages after reconnected. This test guarantees that the * internal producer will continuously resent messages even though the client side encounters the following bugs. From 3cf56d8e1edf0d2d588c9236018653a23649c2db Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 1 Jul 2026 02:47:12 +0300 Subject: [PATCH 086/213] [fix][broker] Fix replication stall when a cursor rewind skips an in-flight read (#26106) (cherry picked from commit baf22113cd785545851f8fe6cfe758692859b867) --- .../persistent/PersistentReplicator.java | 38 +- .../PersistentReplicatorInflightTaskTest.java | 330 ++++++++++++++++++ 2 files changed, 366 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java index 8a8fa0bf74d59..c1411592ddd07 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java @@ -389,13 +389,47 @@ public void readEntriesComplete(List entries, Object ctx) { InFlightTask inFlightTask = (InFlightTask) ctx; latestPublishTime = System.currentTimeMillis(); - // Release memory if terminated. + // The read result must be discarded because the replicator is terminating or the cursor was + // rewound while this read was in flight (e.g. after a failed publish or a schema change). if (state == State.Terminated || state == State.Terminating || inFlightTask.isSkipReadResultDueToCursorRewind()) { for (Entry entry : entries) { - inFlightTask.incCompletedEntries(); entry.release(); } + boolean resumeReads = state != State.Terminated && state != State.Terminating; + // Hold the inFlightTasks lock (the same lock doRewindCursor() and readMoreEntries()'s + // read-scheduling path use) so completing the task and rewinding the cursor here are atomic with + // respect to readMoreEntries(), which reads both the free read slot and the cursor read position + // under that lock; otherwise a concurrent read could observe the freed slot but the stale + // (advanced) read position. + synchronized (inFlightTasks) { + if (resumeReads) { + // The discarded read already advanced the cursor read position past these still-unacked + // messages (ManagedCursor advances the read position when a read completes), which + // overwrites the earlier doRewindCursor(). Rewind again so the messages are re-read; + // otherwise they stay stranded behind the read position and the backlog never drains. + // rewind() resets the read position to the mark-delete position, so only unacked + // messages are re-read; entries already acknowledged (replicated) are skipped on the + // re-read. This makes the recovery at-least-once, with duplicates bounded to the + // messages that were in flight when the rewind happened. + cursor.rewind(); + } + // Complete the task with an empty result so it releases its permit and no longer counts + // as a pending cursor read. Without this, a pending read that could not be cancelled + // (cursor.cancelPendingReadRequest() returns false for an already-dispatched read, which + // is the common case when there is a backlog) would stay entries == null forever, keeping + // hasPendingRead() permanently true and stalling replication. + if (inFlightTask.getEntries() == null) { + inFlightTask.setEntries(Collections.emptyList()); + } + } + if (resumeReads) { + // Resume reading on the broker executor rather than calling readMoreEntries() directly: + // cursor reads can complete inline (cache hit), so a direct call risks deep + // readEntriesComplete -> readMoreEntries recursion (StackOverflowError), the same hazard + // PersistentDispatcherMultipleConsumers.readMoreEntriesAsync() guards against. + topic.getBrokerService().executor().execute(this::readMoreEntries); + } return; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java index a34e86fd11837..63f7abbd640cc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java @@ -25,6 +25,7 @@ import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; @@ -33,16 +34,20 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import io.netty.channel.EventLoopGroup; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Entry; @@ -63,11 +68,14 @@ import org.apache.pulsar.broker.service.persistent.PersistentReplicator.ProducerSendCallback; import org.apache.pulsar.broker.service.persistent.PersistentReplicator.ReasonOfWaitForCursorRewinding; import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerBuilder; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.awaitility.Awaitility; import org.mockito.invocation.InvocationOnMock; @@ -219,6 +227,328 @@ public void testFailedPublishCompletesInFlightTask() throws Exception { } } + /** + * Reproduces a geo-replication stall on the cursor-rewind path. + * + *

When a cursor rewind happens while a cursor read has already been dispatched to bookies, + * {@code cursor.cancelPendingReadRequest()} returns {@code false} (there is no registered waiting + * read op to cancel), so {@code cancelPendingReadTasks()} only flags the in-flight task with + * {@code skipReadResultDueToCursorRewind=true} without completing it. When that dispatched read + * later completes, {@link PersistentReplicator#readEntriesComplete} hits the skip branch and must + * still complete the task; otherwise the task stays {@code entries == null} forever, + * {@link PersistentReplicator#hasPendingRead()} stays {@code true}, all permits remain occupied, + * and reads never resume — replication stalls with backlog. + */ + @Test + public void testCursorRewindSkippedReadCompletesInFlightTask() throws Exception { + PersistentReplicator replicator = spy(getReplicator(topicName)); + // Isolate the unit: don't issue a real cursor read, only verify reads are resumed. + doNothing().when(replicator).readMoreEntries(); + + LinkedList inFlightTasks = replicator.inFlightTasks; + List originalTasks = new ArrayList<>(inFlightTasks); + inFlightTasks.clear(); + + try { + int fullPermits = replicator.getPermitsIfNoPendingRead(); + assertTrue(fullPermits > 0, "precondition: replicator should have free permits"); + + // A pending cursor read (entries == null) flagged to be skipped because of a cursor rewind + // whose pending read could not be cancelled (cancelPendingReadRequest() returned false). + InFlightTask task = + new InFlightTask(PositionFactory.create(1, 1), 1, replicator.getReplicatorId()); + task.setSkipReadResultDueToCursorRewind(true); + inFlightTasks.add(task); + + // Precondition: the uncompleted task blocks all reads. + assertTrue(replicator.hasPendingRead(), "precondition: task must look like a pending read"); + assertEquals(replicator.getPermitsIfNoPendingRead(), 0, + "precondition: pending read must occupy all permits"); + + // The dispatched read finally completes; its result is discarded because of the rewind. + replicator.readEntriesComplete(Collections.singletonList(mock(Entry.class)), task); + + // The task must be completed so it no longer blocks replication. + assertTrue(task.isDone(), "skipped read must complete the in-flight task"); + assertFalse(replicator.hasPendingRead(), + "replication must not stay stuck on an uncompleted pending read"); + assertEquals(replicator.getPermitsIfNoPendingRead(), fullPermits, + "permits must be released after the skipped read completes"); + // Reads must be resumed once the slot is freed (dispatched on the broker executor to + // avoid recursing in the read-completion thread). + Awaitility.await().untilAsserted(() -> verify(replicator, atLeastOnce()).readMoreEntries()); + } finally { + inFlightTasks.clear(); + inFlightTasks.addAll(originalTasks); + } + } + + /** + * End-to-end reproduction of the cursor-rewind stall over a real two-cluster replication setup. + * + *

A real cursor read is held in flight (entries == null) while the cursor is rewound the same way + * {@link ProducerSendCallback#sendComplete} does on a failed publish to the remote cluster: + * {@code beforeTerminateOrCursorRewinding(Failed_Publishing)} followed by {@code doRewindCursor(false)}. + * Because the read was already dispatched, {@code cursor.cancelPendingReadRequest()} returns false, so + * {@code cancelPendingReadTasks()} only flags the task and does not complete it. When the dispatched + * read then completes through the skip branch, the task must still be completed and reads resumed — + * otherwise the replicator stalls and the backlog is never delivered to the remote cluster. + * + *

Before the fix this test times out: the task stays {@code entries == null}, {@code hasPendingRead()} + * stays true, and the backlog never drains. + */ + @Test + public void testReplicationRecoversAfterPublishFailureRewindWithInflightRead() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + nonReplicatedNamespace + "/tp_"); + final int messageCount = 5; + final CountDownLatch readBlocked = new CountDownLatch(1); + final CountDownLatch releaseRead = new CountDownLatch(1); + final AtomicBoolean blockNextRead = new AtomicBoolean(true); + Producer producer = null; + Consumer remoteConsumer = null; + try { + admin1.topics().createNonPartitionedTopic(topicName); + admin2.topics().createNonPartitionedTopic(topicName); + // Create the verifier subscription on the remote topic up front so replicated messages are + // retained for the end-to-end assertion regardless of retention policy. + admin2.topics().createSubscription(topicName, "e2e-verify", MessageId.earliest); + + // Produce a backlog before replication starts so the replicator must read it from the ledger. + producer = client1.newProducer(Schema.STRING).topic(topicName).enableBatching(false).create(); + for (int i = 0; i < messageCount; i++) { + producer.send("msg-" + i); + } + + PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopic(topicName, false) + .join().get(); + ManagedLedgerImpl ml = (ManagedLedgerImpl) topic.getManagedLedger(); + // Clear the entry cache and intercept ledger reads: block the first read so it stays in flight + // (the in-flight task keeps entries == null), then let it and all later reads succeed. + ManagedLedgerTest.makeReadEntryProbFail(ml, () -> { + if (blockNextRead.compareAndSet(true, false)) { + readBlocked.countDown(); + try { + releaseRead.await(30, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + // Never fail the read; it must complete successfully to reach the skip branch. + return null; + }); + + // Start replication from earliest; the first read blocks inside the ledger read (in flight). + pulsar1.getConfig().setReplicationStartAt("earliest"); + admin1.topics().setReplicationClusters(topicName, Arrays.asList(cluster1, cluster2)); + assertTrue(readBlocked.await(30, TimeUnit.SECONDS), "the replicator's cursor read should start"); + + PersistentReplicator replicator = (PersistentReplicator) topic.getReplicators().get(cluster2); + Assert.assertNotNull(replicator, "Replicator should not be null"); + // The dispatched read is in flight and occupies the only read slot. + assertTrue(replicator.hasPendingRead()); + + // Rewind the cursor exactly as a failed publish to the remote cluster does. The in-flight read + // was already dispatched, so cancelPendingReadRequest() returns false and the task is only + // flagged for skipping (not completed). + replicator.beforeTerminateOrCursorRewinding(ReasonOfWaitForCursorRewinding.Failed_Publishing); + replicator.doRewindCursor(false); + + // The in-flight read now completes successfully and is discarded by the skip branch. + releaseRead.countDown(); + + // Recovery: the freed slot lets reading resume, so the whole backlog is replicated and the + // cursor mark-delete advances, draining the backlog to 0. Before the fix the leaked task keeps + // getPermitsIfNoPendingRead() == 0, so no read is ever issued and the backlog stays at + // messageCount (this await times out). Note: hasPendingRead() is intentionally NOT used as the + // recovery signal because a healthy idle replicator also holds a pending "wait for new entries" + // read, so it stays true even after a successful recovery. + Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> + assertEquals(replicator.getNumberOfEntriesInBacklog(), 0, + "replication backlog must drain after recovery")); + + // End-to-end: verify every produced message is delivered to the remote cluster, with no loss + // and (in this single-batch scenario, where the discarded read sent nothing) no duplicates. + remoteConsumer = client2.newConsumer(Schema.STRING).topic(topicName) + .subscriptionName("e2e-verify") + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + List deliveredValues = new ArrayList<>(); + for (int i = 0; i < messageCount; i++) { + Message received = remoteConsumer.receive(30, TimeUnit.SECONDS); + Assert.assertNotNull(received, "remote cluster should receive replicated message " + i); + deliveredValues.add(received.getValue()); + remoteConsumer.acknowledge(received); + } + // No duplicate/extra messages were replicated on the rewind/recovery path. + Assert.assertNull(remoteConsumer.receive(3, TimeUnit.SECONDS), + "no duplicate messages should be replicated after recovery"); + Set expectedValues = new HashSet<>(); + for (int i = 0; i < messageCount; i++) { + expectedValues.add("msg-" + i); + } + assertEquals(new HashSet<>(deliveredValues), expectedValues, + "every produced message must be replicated exactly once (no loss)"); + } finally { + releaseRead.countDown(); + if (producer != null) { + producer.close(); + } + if (remoteConsumer != null) { + remoteConsumer.close(); + } + admin1.topics().delete(topicName, true); + admin2.topics().delete(topicName, true); + } + } + + /** + * End-to-end reproduction of the cursor-rewind stall on the schema-fetch path. + * + *

{@link GeoPersistentReplicator#replicateEntries} rewinds the cursor when it reaches a message + * whose schema is not yet available locally: it calls + * {@code beforeTerminateOrCursorRewinding(Fetching_Schema)} synchronously and, once the schema has + * been fetched, {@code doRewindCursor(true)} (which rewinds the cursor and resumes reads). If a + * cursor read was already dispatched when the schema-fetch rewind happens, + * {@code cursor.cancelPendingReadRequest()} returns false, so the in-flight task is only flagged + * for skipping and is not completed. When that dispatched read later completes through the + * {@link PersistentReplicator#readEntriesComplete} skip branch it must still be completed and reads + * resumed; otherwise the task stays {@code entries == null}, {@link + * PersistentReplicator#hasPendingRead()} stays true, all permits remain occupied and replication + * stalls with backlog. This is the same root cause as + * {@link #testReplicationRecoversAfterPublishFailureRewindWithInflightRead}, reached through the + * {@code Fetching_Schema} rewind reason (with {@code doRewindCursor(true)}) instead of + * {@code Failed_Publishing}; it is the stall observed as the flaky + * {@code ReplicatorTest.testReplicationWithSchema}. + * + *

The schema-fetch rewind is driven directly (rather than by crossing a real schema boundary) + * because the bug requires a cursor read to be in flight at the exact moment the rewind runs. On + * the natural path that needs a read to be pipelined — dispatched by an earlier message's + * {@code sendComplete} — concurrently with {@code replicateEntries} reaching the schema boundary, + * an inherently racy interleaving that cannot be reproduced deterministically (which is why + * {@code testReplicationWithSchema} only fails intermittently). This test issues the same two + * rewind calls ({@code beforeTerminateOrCursorRewinding(Fetching_Schema)} then + * {@code doRewindCursor(true)}) that {@code GeoPersistentReplicator.replicateEntries} issues at a + * schema boundary; in production they are separated by the asynchronous schema fetch and the + * stranded read is a separately-pipelined one, so this test collapses that timing into a + * deterministic sequence on a single held-in-flight read. It therefore guards the + * {@code readEntriesComplete} skip-branch recovery for the {@code Fetching_Schema} rewind, not the + * schema-detection wiring in {@code replicateEntries} itself. + * + *

Before the fix this test times out: the task stays {@code entries == null} and the backlog + * never drains. + */ + @Test + public void testReplicationRecoversAfterSchemaFetchRewindWithInflightRead() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + nonReplicatedNamespace + "/tp_"); + final int messageCount = 5; + final CountDownLatch readBlocked = new CountDownLatch(1); + final CountDownLatch releaseRead = new CountDownLatch(1); + final AtomicBoolean blockNextRead = new AtomicBoolean(true); + // Capture and restore the shared per-class broker config so this method does not leak + // "earliest" into sibling tests that rely on the default replicationStartAt. + final String prevReplicationStartAt = pulsar1.getConfig().getReplicationStartAt(); + Producer producer = null; + Consumer remoteConsumer = null; + try { + admin1.topics().createNonPartitionedTopic(topicName); + admin2.topics().createNonPartitionedTopic(topicName); + // Create the verifier subscription on the remote topic up front so replicated messages are + // retained for the end-to-end assertion regardless of retention policy. + admin2.topics().createSubscription(topicName, "e2e-verify", MessageId.earliest); + + // Produce a backlog before replication starts so the replicator must read it from the ledger. + producer = client1.newProducer(Schema.STRING).topic(topicName).enableBatching(false).create(); + for (int i = 0; i < messageCount; i++) { + producer.send("msg-" + i); + } + + PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopic(topicName, false) + .join().get(); + ManagedLedgerImpl ml = (ManagedLedgerImpl) topic.getManagedLedger(); + // Clear the entry cache and intercept ledger reads: block the first read so it stays in flight + // (the in-flight task keeps entries == null), then let it and all later reads succeed. + ManagedLedgerTest.makeReadEntryProbFail(ml, () -> { + if (blockNextRead.compareAndSet(true, false)) { + readBlocked.countDown(); + try { + releaseRead.await(30, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + // Never fail the read; it must complete successfully to reach the skip branch. + return null; + }); + + // Start replication from earliest; the first read blocks inside the ledger read (in flight). + pulsar1.getConfig().setReplicationStartAt("earliest"); + admin1.topics().setReplicationClusters(topicName, Arrays.asList(cluster1, cluster2)); + assertTrue(readBlocked.await(30, TimeUnit.SECONDS), "the replicator's cursor read should start"); + + PersistentReplicator replicator = (PersistentReplicator) topic.getReplicators().get(cluster2); + Assert.assertNotNull(replicator, "Replicator should not be null"); + // The dispatched read is in flight and occupies the only read slot. + assertTrue(replicator.hasPendingRead()); + + // Rewind the cursor with the same two calls GeoPersistentReplicator.replicateEntries issues + // when it reaches a message whose schema must be fetched from the local cluster: flag the + // in-flight read for skipping (beforeTerminateOrCursorRewinding(Fetching_Schema)), then, once + // the schema has been fetched, rewind and resume reads (doRewindCursor(true)). The in-flight + // read was already dispatched, so cancelPendingReadRequest() returns false and the task is only + // flagged, not completed. doRewindCursor(true)'s readMoreEntries() is a no-op here because the + // still-pending (skip-flagged) read keeps hasPendingRead() true. + replicator.beforeTerminateOrCursorRewinding(ReasonOfWaitForCursorRewinding.Fetching_Schema); + replicator.doRewindCursor(true); + + // The in-flight read now completes successfully and is discarded by the skip branch. + releaseRead.countDown(); + + // Recovery: the freed slot lets reading resume, so the whole backlog is replicated and the + // cursor mark-delete advances, draining the backlog to 0. Before the fix the leaked task keeps + // getPermitsIfNoPendingRead() == 0, so no read is ever issued and the backlog stays at + // messageCount (this await times out). Note: hasPendingRead() is intentionally NOT used as the + // recovery signal because a healthy idle replicator also holds a pending "wait for new entries" + // read, so it stays true even after a successful recovery. + Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> + assertEquals(replicator.getNumberOfEntriesInBacklog(), 0, + "replication backlog must drain after recovery")); + + // End-to-end: verify every produced message is delivered to the remote cluster, with no loss + // and (in this single-batch scenario, where the discarded read sent nothing) no duplicates. + remoteConsumer = client2.newConsumer(Schema.STRING).topic(topicName) + .subscriptionName("e2e-verify") + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + List deliveredValues = new ArrayList<>(); + for (int i = 0; i < messageCount; i++) { + Message received = remoteConsumer.receive(30, TimeUnit.SECONDS); + Assert.assertNotNull(received, "remote cluster should receive replicated message " + i); + deliveredValues.add(received.getValue()); + remoteConsumer.acknowledge(received); + } + // No duplicate/extra messages were replicated on the rewind/recovery path. + Assert.assertNull(remoteConsumer.receive(3, TimeUnit.SECONDS), + "no duplicate messages should be replicated after recovery"); + Set expectedValues = new HashSet<>(); + for (int i = 0; i < messageCount; i++) { + expectedValues.add("msg-" + i); + } + assertEquals(new HashSet<>(deliveredValues), expectedValues, + "every produced message must be replicated exactly once (no loss)"); + } finally { + releaseRead.countDown(); + pulsar1.getConfig().setReplicationStartAt(prevReplicationStartAt); + if (producer != null) { + producer.close(); + } + if (remoteConsumer != null) { + remoteConsumer.close(); + } + admin1.topics().delete(topicName, true); + admin2.topics().delete(topicName, true); + } + } + @DataProvider public Object[][] readSchedulingLimits() { return new Object[][] { From c797c36bb203edeea06de3a53854b9616ecee5b9 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 1 Jul 2026 03:51:27 +0300 Subject: [PATCH 087/213] [fix][test] Fix flaky AuditorBookieTest.testBookieClusterRestart (#26122) (cherry picked from commit 116212f5424d05da7affb078308d003d5e180423) --- .../bookkeeper/replication/AuditorBookieTest.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieTest.java index 14cdf3e1fc29c..db54196321b82 100644 --- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieTest.java +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieTest.java @@ -138,13 +138,12 @@ public void testBookieClusterRestart() throws Exception { assertTrue("Auditor elector is not running!", auditorElector .isRunning()); } - stopBKCluster(); stopAuditorElectors(); - - startBKCluster(zkUtil.getMetadataServiceUri()); - //startBKCluster(zkUtil.getMetadataServiceUri()) override the base conf metadataServiceUri - baseConf.setMetadataServiceUri( - zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + // Restart the bookies while preserving their identities (host:port and data dirs). + // Tearing the cluster down and recreating bookies with fresh data dirs on recycled + // ports fails bookie cookie validation against the cookies that are still registered + // in the metadata store, which made this test flaky. + restartBookies(); startAuditorElectors(); BookieServer newAuditor = waitForNewAuditor(auditor); assertNotSame( @@ -230,6 +229,10 @@ private void stopAuditorElectors() throws Exception { LOG.debug("Stopping Auditor Elector!"); } } + // The same test instance is reused across test methods, so drop references to the + // shut-down electors. Otherwise a later method that iterates over auditorElectors + // (e.g. testBookieClusterRestart) would observe stale, already-stopped electors. + auditorElectors.clear(); } private BookieServer verifyAuditor() throws Exception { From d47c4b3ee53b440617fa9ec07cb748a08c678145 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Fri, 1 May 2026 10:32:26 -0700 Subject: [PATCH 088/213] [fix][test] Fix flaky SchemaServiceTest.testSchemaRegistryMetrics (#25645) (cherry picked from commit c3fde128619e562d70c77f9a242242a9617d6d3b) --- .../broker/service/schema/SchemaServiceTest.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java index d164559e85858..757dfd827ff52 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java @@ -173,12 +173,16 @@ public void testSchemaRegistryMetrics() throws Exception { String metricsStr = output.toString(StandardCharsets.UTF_8); Multimap metrics = parseMetrics(metricsStr); + // The *_ops_failed_total counters are registered on the default Prometheus + // registry (a JVM-wide static), so labels accumulated by other tests in the + // same JVM persist here. Only assert that no failed metric exists for THIS + // test's namespace. Collection delMetrics = metrics.get("pulsar_schema_del_ops_failed_total"); - Assert.assertEquals(delMetrics.size(), 0); + assertThat(delMetrics).noneMatch(metric -> namespace.equals(metric.tags.get("namespace"))); Collection getMetrics = metrics.get("pulsar_schema_get_ops_failed_total"); - Assert.assertEquals(getMetrics.size(), 0); + assertThat(getMetrics).noneMatch(metric -> namespace.equals(metric.tags.get("namespace"))); Collection putMetrics = metrics.get("pulsar_schema_put_ops_failed_total"); - Assert.assertEquals(putMetrics.size(), 0); + assertThat(putMetrics).noneMatch(metric -> namespace.equals(metric.tags.get("namespace"))); Collection deleteLatency = metrics.get("pulsar_schema_del_ops_latency_count"); assertThat(deleteLatency).anySatisfy(metric -> { From 29fcd20574539bf6716f37ff435b77ecf24921bc Mon Sep 17 00:00:00 2001 From: void-ptr974 Date: Wed, 1 Jul 2026 08:53:02 +0800 Subject: [PATCH 089/213] [fix][test] Fix flaky PersistentTopicsTest setup caused by concurrent Mockito stubbing (#26083) (cherry picked from commit aa2d1bc61f0fe82b50848ebcbb569992ef48918d) --- .../broker/admin/PersistentTopicsTest.java | 49 +++++++++++++------ 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index 40df6b0b5ffe1..626805a7032a7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -30,7 +30,6 @@ import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertSame; @@ -48,6 +47,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; import javax.servlet.ServletContext; import javax.ws.rs.InternalServerErrorException; import javax.ws.rs.WebApplicationException; @@ -58,6 +58,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.admin.v2.ExtPersistentTopics; import org.apache.pulsar.broker.admin.v2.NonPersistentTopics; @@ -129,7 +130,8 @@ public class PersistentTopicsTest extends MockedPulsarServiceBaseTest { protected Field uriField; protected UriInfo uriInfo; private NonPersistentTopics nonPersistentTopic; - private NamespaceResources namespaceResources; + private volatile NamespaceResources namespaceResourcesOverride; + private volatile Function>> listPersistentTopicsAsyncHandler; @BeforeClass public void initPersistentTopics() throws Exception { @@ -172,7 +174,6 @@ protected void setup() throws Exception { nonPersistentTopic = spy(NonPersistentTopics.class); nonPersistentTopic.setServletContext(mock(ServletContext.class)); nonPersistentTopic.setPulsar(pulsar); - namespaceResources = mock(NamespaceResources.class); doReturn(false).when(nonPersistentTopic).isRequestHttps(); doReturn(null).when(nonPersistentTopic).originalPrincipal(); doReturn("test").when(nonPersistentTopic).clientAppId(); @@ -180,10 +181,27 @@ protected void setup() throws Exception { doNothing().when(nonPersistentTopic).validateAdminAccessForTenant(this.testTenant); doReturn(mock(AuthenticationDataHttps.class)).when(nonPersistentTopic).clientAuthData(); - PulsarResources resources = - spy(new PulsarResources(pulsar.getLocalMetadataStore(), pulsar.getConfigurationMetadataStore())); - doReturn(spy(new TopicResources(pulsar.getLocalMetadataStore()))).when(resources).getTopicResources(); - doReturn(resources).when(pulsar).getPulsarResources(); + TopicResources topicResources = BrokerTestUtil.spyWithoutRecordingInvocations( + new TopicResources(pulsar.getLocalMetadataStore())); + PulsarResources resources = BrokerTestUtil.spyWithoutRecordingInvocations( + new PulsarResources(pulsar.getLocalMetadataStore(), pulsar.getConfigurationMetadataStore())); + NamespaceResources namespaceResources = resources.getNamespaceResources(); + doAnswer(invocation -> { + NamespaceResources override = namespaceResourcesOverride; + return override != null ? override : namespaceResources; + }).when(resources).getNamespaceResources(); + doReturn(topicResources).when(resources).getTopicResources(); + doAnswer(invocation -> { + Function>> handler = listPersistentTopicsAsyncHandler; + if (handler != null) { + CompletableFuture> result = handler.apply(invocation.getArgument(0)); + if (result != null) { + return result; + } + } + return invocation.callRealMethod(); + }).when(topicResources).listPersistentTopicsAsync(any()); + FieldUtils.writeField(pulsar, "pulsarResources", resources, true); admin.clusters().createCluster("use", ClusterData.builder().serviceUrl("http://127.0.0.3:8082").build()); admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); @@ -199,6 +217,8 @@ protected void setup() throws Exception { @Override @AfterMethod(alwaysRun = true) protected void cleanup() throws Exception { + namespaceResourcesOverride = null; + listPersistentTopicsAsyncHandler = null; super.internalCleanup(); } @@ -606,8 +626,8 @@ public void testCreateTopicWithReplicationCluster() { CompletableFuture> policyFuture = new CompletableFuture<>(); Policies policies = new Policies(); policyFuture.complete(Optional.of(policies)); - when(pulsar.getPulsarResources().getNamespaceResources()).thenReturn(namespaceResources); - doReturn(policyFuture).when(namespaceResources).getPoliciesAsync(namespaceName); + namespaceResourcesOverride = mock(NamespaceResources.class); + doReturn(policyFuture).when(namespaceResourcesOverride).getPoliciesAsync(namespaceName); AsyncResponse response = mock(AsyncResponse.class); ArgumentCaptor errCaptor = ArgumentCaptor.forClass(RestException.class); persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName, 2, true); @@ -619,7 +639,7 @@ public void testCreateTopicWithReplicationCluster() { // Test policy not exist and return 'Namespace not found' CompletableFuture> policyFuture2 = new CompletableFuture<>(); policyFuture2.complete(Optional.empty()); - doReturn(policyFuture2).when(namespaceResources).getPoliciesAsync(namespaceName); + doReturn(policyFuture2).when(namespaceResourcesOverride).getPoliciesAsync(namespaceName); response = mock(AsyncResponse.class); errCaptor = ArgumentCaptor.forClass(RestException.class); persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName, 2, true); @@ -653,12 +673,13 @@ public void testCreatePartitionedTopicHavingNonPartitionTopicWithPartitionSuffix final String nonPartitionTopicName2 = "special-topic-partition-123"; final String partitionedTopicName = "special-topic"; - when(pulsar.getPulsarResources().getTopicResources() - .listPersistentTopicsAsync(NamespaceName.get("my-tenant/my-namespace"))) - .thenReturn(CompletableFuture.completedFuture(List.of( + NamespaceName namespaceName = NamespaceName.get("my-tenant/my-namespace"); + listPersistentTopicsAsyncHandler = namespace -> namespaceName.equals(namespace) + ? CompletableFuture.completedFuture(List.of( "persistent://my-tenant/my-namespace/" + nonPartitionTopicName1, "persistent://my-tenant/my-namespace/" + nonPartitionTopicName2 - ))); + )) + : null; // doReturn(ImmutableSet.of(nonPartitionTopicName1, nonPartitionTopicName2)).when(mockZooKeeperChildrenCache) // .get(anyString()); // doReturn(CompletableFuture.completedFuture(ImmutableSet.of(nonPartitionTopicName1, nonPartitionTopicName2)) From c2aad8d1823826d8bf17bfc20207b31f290d3608 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 1 Jul 2026 06:32:42 +0300 Subject: [PATCH 090/213] [fix][test] Run makeReadEntryProbFail's errorOrNot on a caller-provided executor (#26123) (cherry picked from commit 12d8aa9a9773f2546f58aca4678c65f6d506dd95) --- .../mledger/impl/ManagedLedgerTest.java | 21 +++++++++++-------- .../broker/service/OneWayReplicatorTest.java | 4 +++- .../PersistentReplicatorInflightTaskTest.java | 21 ++++++++++++++++--- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java index 4f302af34d54f..1da9980585cf5 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java @@ -78,6 +78,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.FutureTask; @@ -190,19 +191,21 @@ public static void makeAddEntryTimeout(ManagedLedgerImpl ml, AtomicBoolean addEn ml.currentLedger = spyLedgerHandle; } - public static void makeReadEntryProbFail(ManagedLedgerImpl ml, Supplier errorOrNot) - throws Exception { + public static void makeReadEntryProbFail(ManagedLedgerImpl ml, Supplier errorOrNot, + Executor errorSupplierExecutor) throws Exception { ml.entryCache.clear(); LedgerHandle currentLedger = ml.currentLedger; final LedgerHandle spyLedgerHandle = spy(currentLedger); doAnswer(invocation -> { - long ledgerId = (long) invocation.getArguments()[0]; - long entryId = (long) invocation.getArguments()[1]; - ManagedLedgerException mightError = errorOrNot.get(); - if (mightError != null) { - return CompletableFuture.failedFuture(mightError); - } - return currentLedger.readUnconfirmedAsync(ledgerId, entryId); + long ledgerId = invocation.getArgument(0); + long entryId = invocation.getArgument(1); + // Evaluate errorOrNot on errorSupplierExecutor. Pass a single-threaded executor when errorOrNot may + // block (e.g. it waits on a CountDownLatch) so it doesn't block the calling read thread; pass + // MoreExecutors.directExecutor() to evaluate it inline on the calling thread. + return CompletableFuture.supplyAsync(errorOrNot, errorSupplierExecutor) + .thenCompose(mightError -> mightError != null + ? CompletableFuture.failedFuture(mightError) + : currentLedger.readUnconfirmedAsync(ledgerId, entryId)); }).when(spyLedgerHandle).readUnconfirmedAsync(anyLong(), anyLong()); ml.currentLedger = spyLedgerHandle; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java index 5e6a78305da8d..99f608a200c34 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java @@ -35,6 +35,7 @@ import com.github.benmanes.caffeine.cache.AsyncLoadingCache; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; +import com.google.common.util.concurrent.MoreExecutors; import io.netty.channel.Channel; import io.netty.util.concurrent.FastThreadLocalThread; import java.lang.reflect.Field; @@ -723,7 +724,8 @@ public void testProbBKErrorWhenReplicating() throws Exception { } return new ManagedLedgerException.TooManyRequestsException("mocked error"); }; - ManagedLedgerTest.makeReadEntryProbFail(ml1, bkErrorOrNot); + // bkErrorOrNot doesn't block, so evaluate it inline on the calling read thread via directExecutor(). + ManagedLedgerTest.makeReadEntryProbFail(ml1, bkErrorOrNot, MoreExecutors.directExecutor()); // Verify: the replication will finish even though received ManagedLedgerException.TooManyRequestsException. pulsar1.getConfig().setReplicationStartAt("earliest"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java index 63f7abbd640cc..4bf96d37b8a21 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentReplicatorInflightTaskTest.java @@ -46,9 +46,12 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedCursor; @@ -162,6 +165,10 @@ public void testReadEntriesFailedCompletesInFlightTaskAfterReplicatorTerminated( PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopic(topicName, false) .join().get(); ManagedLedgerImpl ml = (ManagedLedgerImpl) topic.getManagedLedger(); + // errorOrNot blocks on failRead, so run it on a dedicated single-threaded executor instead of the + // calling read thread; otherwise the blocked read thread would stall replicator.terminate(). + @Cleanup("shutdownNow") + ExecutorService readFailExecutor = Executors.newSingleThreadExecutor(); ManagedLedgerTest.makeReadEntryProbFail(ml, () -> { readStarted.countDown(); try { @@ -173,7 +180,7 @@ public void testReadEntriesFailedCompletesInFlightTaskAfterReplicatorTerminated( return new ManagedLedgerException(e); } return new ManagedLedgerException.TooManyRequestsException("mocked read failure"); - }); + }, readFailExecutor); pulsar1.getConfig().setReplicationStartAt("earliest"); admin1.topics().setReplicationClusters(topicName, Arrays.asList(cluster1, cluster2)); @@ -324,6 +331,10 @@ public void testReplicationRecoversAfterPublishFailureRewindWithInflightRead() t ManagedLedgerImpl ml = (ManagedLedgerImpl) topic.getManagedLedger(); // Clear the entry cache and intercept ledger reads: block the first read so it stays in flight // (the in-flight task keeps entries == null), then let it and all later reads succeed. + // errorOrNot blocks the first read on releaseRead, so run it on a dedicated single-threaded executor + // instead of the calling read thread. + @Cleanup("shutdownNow") + ExecutorService readFailExecutor = Executors.newSingleThreadExecutor(); ManagedLedgerTest.makeReadEntryProbFail(ml, () -> { if (blockNextRead.compareAndSet(true, false)) { readBlocked.countDown(); @@ -335,7 +346,7 @@ public void testReplicationRecoversAfterPublishFailureRewindWithInflightRead() t } // Never fail the read; it must complete successfully to reach the skip branch. return null; - }); + }, readFailExecutor); // Start replication from earliest; the first read blocks inside the ledger read (in flight). pulsar1.getConfig().setReplicationStartAt("earliest"); @@ -467,6 +478,10 @@ public void testReplicationRecoversAfterSchemaFetchRewindWithInflightRead() thro ManagedLedgerImpl ml = (ManagedLedgerImpl) topic.getManagedLedger(); // Clear the entry cache and intercept ledger reads: block the first read so it stays in flight // (the in-flight task keeps entries == null), then let it and all later reads succeed. + // errorOrNot blocks the first read on releaseRead, so run it on a dedicated single-threaded executor + // instead of the calling read thread. + @Cleanup("shutdownNow") + ExecutorService readFailExecutor = Executors.newSingleThreadExecutor(); ManagedLedgerTest.makeReadEntryProbFail(ml, () -> { if (blockNextRead.compareAndSet(true, false)) { readBlocked.countDown(); @@ -478,7 +493,7 @@ public void testReplicationRecoversAfterSchemaFetchRewindWithInflightRead() thro } // Never fail the read; it must complete successfully to reach the skip branch. return null; - }); + }, readFailExecutor); // Start replication from earliest; the first read blocks inside the ledger read (in flight). pulsar1.getConfig().setReplicationStartAt("earliest"); From 50c4c934aa52471a0b50b9709c08d442ebb62f46 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Jul 2026 03:02:34 +0300 Subject: [PATCH 091/213] [fix][broker] Don't let a closing topic-policies reader abort a concurrent cache-init reload (#26132) (cherry picked from commit decc80ff0369c0387c6cd02745ebe993dbf71ff1) --- .../SystemTopicBasedTopicPoliciesService.java | 108 +++++++----------- ...temTopicBasedTopicPoliciesServiceTest.java | 93 ++++++++++++--- 2 files changed, 116 insertions(+), 85 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index 6c847e0867d74..3fdfcc7b0ad91 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -462,7 +462,7 @@ private CompletableFuture sendTopicPolicyEventInternal(TopicName topi // The cached writer will be closed when an exception happens // This is potentially not a great idea since we should be able to rely on the Pulsar client's // behavior for restoring a Producer after a failure. - writerCaches.synchronous().invalidate(topicName.getNamespaceObject()); + cleanWriterCache(topicName.getNamespaceObject()); throw FutureUtil.wrapToCompletionException(t); }); } @@ -653,7 +653,7 @@ public void addOwnedNamespaceBundleAsync(NamespaceBundle namespaceBundle) { initPolicesCache(reader, stageFuture); return stageFuture // Read policies in background - .thenAccept(__ -> readMorePoliciesAsync(reader)); + .thenAccept(__ -> readMorePoliciesAsync(reader, initNamespacePolicyFuture)); }).thenApply(__ -> { initNamespacePolicyFuture.complete(null); return null; @@ -689,13 +689,7 @@ public void addOwnedNamespaceBundleAsync(NamespaceBundle namespaceBundle) { } private CompletableFuture> newReader(NamespaceName ns) { - return readerCaches.compute(ns, (__, existingFuture) -> { - if (existingFuture == null) { - return createSystemTopicClient(ns); - } - - return existingFuture; - }); + return readerCaches.computeIfAbsent(ns, __ -> createSystemTopicClient(ns)); } protected CompletableFuture> createSystemTopicClient( @@ -721,7 +715,7 @@ void removeOwnedNamespaceBundleAsync(NamespaceBundle namespaceBundle) { } AtomicInteger bundlesCount = ownedBundlesCountPerNamespace.get(namespace); if (bundlesCount == null || bundlesCount.decrementAndGet() <= 0) { - cleanPoliciesCacheInitMap(namespace, true); + cleanPoliciesCacheInitMap(namespace); cleanWriterCache(namespace); cleanOwnedBundlesCount(namespace); } @@ -786,17 +780,20 @@ private void scheduleInitPolicesCacheTimeout(@NonNull NamespaceName namespace, /** * Identity-guarded cleanup for an initialization that failed (it timed out, the {@code __change_events} reader - * could not be created, or reading the topic threw). Unlike {@link #cleanPoliciesCacheInitMap}, which + * could not be created, or reading the topic threw), or whose background reader was later closed (an + * {@code AlreadyClosedException} surfaced in {@link #readMorePoliciesAsync} after a namespace unload closed the + * reader). Unlike {@link #cleanPoliciesCacheInitMap}, which * removes/closes by namespace key unconditionally, this only tears down state that still belongs to * {@code initFuture}. By the time the failure is observed, a concurrent retry — or a namespace-bundle unload that * left the init future orphaned — may already own the namespace with a fresh future and reader; removing by key * would drop that newer future and close its reader, pinning the namespace again. Guarding on identity ensures a * late failure never clobbers a newer initialization. * - * @param closeReader when {@code true}, also clears the cached policies and closes the reader that belongs to this - * initialization; when {@code false}, only the init future is dropped, leaving the reader cached - * for the retry to reuse (mirrors the transient read-error path of - * {@link #cleanPoliciesCacheInitMap}). + * @param closeReader when {@code true}, also closes the reader and message-handler tracker that belong to this + * initialization; when {@code false}, only the init future is dropped, leaving the reader + * cached for the retry to reuse. The cached policies are intentionally left in place; they + * are cleared only when the whole namespace is unloaded, so this cleanup cannot race a + * concurrent re-initialization. */ @VisibleForTesting void cleanupFailedPolicyCacheInit(@NonNull NamespaceName namespace, @@ -805,23 +802,29 @@ void cleanupFailedPolicyCacheInit(@NonNull NamespaceName namespace, // initialization, never one a concurrent retry creates immediately afterwards. CompletableFuture> readerFuture = closeReader ? readerCaches.get(namespace) : null; + TopicPolicyMessageHandlerTracker tracker = topicPolicyMessageHandlerTrackers.get(namespace); + + // Identity guard: only proceed while this initialization still owns the namespace's init future. if (!policyCacheInitMap.remove(namespace, initFuture)) { // Superseded by a retry or an unload; that owner is responsible for its own reader/state. return; } + // Complete the dropped future (a no-op if the caller already completed it) outside any map remapping function, // so awaiting topic loads fail fast and retry instead of hanging until the broker restarts (issue #25294). failPendingPolicyCacheInit(namespace, initFuture); if (!closeReader) { return; } - policiesCache.entrySet().removeIf(entry -> Objects.equals(entry.getKey().getNamespaceObject(), namespace)); - globalPoliciesCache.entrySet() - .removeIf(entry -> Objects.equals(entry.getKey().getNamespaceObject(), namespace)); - TopicPolicyMessageHandlerTracker tracker = topicPolicyMessageHandlerTrackers.remove(namespace); - if (tracker != null) { + + // Close the tracker captured above only if it is still the one installed for this namespace, so a + // concurrent re-initialization that installed a newer tracker is left untouched. + if (tracker != null && topicPolicyMessageHandlerTrackers.remove(namespace, tracker)) { tracker.close(); } + + // Remove and close the reader captured above only if it is still the current one, so a reader + // created by a later initialization is never closed by this stale cleanup. if (readerFuture != null && readerCaches.remove(namespace, readerFuture) && !readerFuture.isCompletedExceptionally()) { readerFuture.thenCompose(SystemTopicClient.Reader::closeAsync) @@ -835,7 +838,7 @@ void cleanupFailedPolicyCacheInit(@NonNull NamespaceName namespace, private void initPolicesCache(SystemTopicClient.Reader reader, CompletableFuture future) { if (closed.get()) { future.completeExceptionally(new BrokerServiceException(getClass().getName() + " is closed.")); - cleanPoliciesCacheInitMap(reader.getSystemTopic().getTopicName().getNamespaceObject(), true); + cleanPoliciesCacheInitMap(reader.getSystemTopic().getTopicName().getNamespaceObject()); return; } reader.hasMoreEventsAsync().whenComplete((hasMore, ex) -> { @@ -880,13 +883,10 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp }); } + // Full teardown of a namespace's topic-policies state: removes and closes the reader, the message-handler + // tracker, the cached policies and the init future. Used when the whole namespace is unloaded. @VisibleForTesting - void cleanPoliciesCacheInitMap(@NonNull NamespaceName namespace, boolean closeReader) { - if (!closeReader) { - failPendingPolicyCacheInit(namespace, policyCacheInitMap.remove(namespace)); - return; - } - + void cleanPoliciesCacheInitMap(@NonNull NamespaceName namespace) { TopicPolicyMessageHandlerTracker topicPolicyMessageHandlerTracker = topicPolicyMessageHandlerTrackers.remove(namespace); if (topicPolicyMessageHandlerTracker != null) { @@ -938,52 +938,17 @@ private void cleanOwnedBundlesCount(@NonNull NamespaceName namespace) { ownedBundlesCountPerNamespace.remove(namespace); } - - private void cleanCacheAndCloseReader(@NonNull NamespaceName namespace, boolean cleanOwnedBundlesCount, - boolean cleanWriterCache) { - if (cleanWriterCache) { - writerCaches.synchronous().invalidate(namespace); - } - CompletableFuture> readerFuture = readerCaches.remove(namespace); - - TopicPolicyMessageHandlerTracker topicPolicyMessageHandlerTracker = - topicPolicyMessageHandlerTrackers.remove(namespace); - if (topicPolicyMessageHandlerTracker != null) { - topicPolicyMessageHandlerTracker.close(); - } - - if (cleanOwnedBundlesCount) { - ownedBundlesCountPerNamespace.remove(namespace); - } - if (readerFuture != null && !readerFuture.isCompletedExceptionally()) { - readerFuture.thenCompose(SystemTopicClient.Reader::closeAsync) - .exceptionally(ex -> { - log.warn("[{}] Close change_event reader fail.", namespace, ex); - return null; - }); - } - - policyCacheInitMap.compute(namespace, (k, v) -> { - policiesCache.entrySet().removeIf(entry -> Objects.equals(entry.getKey().getNamespaceObject(), namespace)); - globalPoliciesCache.entrySet() - .removeIf(entry -> Objects.equals(entry.getKey().getNamespaceObject(), namespace)); - return null; - }); - } - - - - /** * This is an async method for the background reader to continue syncing new messages. * * Note: You should not do any blocking call here. because it will affect * #{@link SystemTopicBasedTopicPoliciesService#getTopicPoliciesAsync} method to block loading topic. */ - private void readMorePoliciesAsync(SystemTopicClient.Reader reader) { + private void readMorePoliciesAsync(SystemTopicClient.Reader reader, + CompletableFuture initFuture) { NamespaceName namespaceObject = reader.getSystemTopic().getTopicName().getNamespaceObject(); if (closed.get()) { - cleanPoliciesCacheInitMap(namespaceObject, true); + cleanupFailedPolicyCacheInit(namespaceObject, initFuture, true); return; } reader.readNextAsync() @@ -1002,15 +967,22 @@ private void readMorePoliciesAsync(SystemTopicClient.Reader reader) }) .whenComplete((__, ex) -> { if (ex == null) { - readMorePoliciesAsync(reader); + readMorePoliciesAsync(reader, initFuture); } else { if (isAlreadyClosedException(ex)) { log.info("Closing the topic policies reader for {}", reader.getSystemTopic().getTopicName()); - cleanPoliciesCacheInitMap(namespaceObject, true); + // Tear down by init-future identity, not by namespace key: this reader may have been + // closed by a namespace unload while a concurrent reload already installed a fresh + // reader and init future for the same namespace (the close only surfaces here, on the + // client executor, afterwards). A namespace-keyed cleanup would clobber that newer + // generation and abort its init with "...aborted because the cached state was cleared", + // failing the reloading topic. cleanupFailedPolicyCacheInit only tears down state that + // still belongs to this initialization, so a superseded reader's late close is a no-op. + cleanupFailedPolicyCacheInit(namespaceObject, initFuture, true); } else { log.warn("Read more topic polices exception, read again.", ex); - readMorePoliciesAsync(reader); + readMorePoliciesAsync(reader, initFuture); } } }); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java index 04b9bc67e7e0a..8ea225ebf6278 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java @@ -604,17 +604,19 @@ public void testPrepareInitPoliciesCacheAsyncThrowExceptionAfterCreateReader() t }); // Cleanup must run exactly once per trigger and not repeat recursively (in older code it ran 3 times). - // Two failures are triggered here: the reader.close() above drives readMorePoliciesAsync into - // cleanPoliciesCacheInitMap (1x), and the second prepareInitPoliciesCacheAsync fails in initPolicesCache and - // is torn down by the identity-guarded cleanupFailedPolicyCacheInit (1x). + // Two failures are triggered here, and both tear down through the identity-guarded cleanupFailedPolicyCacheInit + // (2x): the reader.close() above drives readMorePoliciesAsync's AlreadyClosed branch into it, and the second + // prepareInitPoliciesCacheAsync fails in initPolicesCache and is torn down by it as well. The namespace-keyed + // cleanPoliciesCacheInitMap must not be reached from the reader-close path, otherwise a superseded reader could + // clobber a newer generation's init future. boolean logFound = testLogAppender.getEvents().stream().anyMatch(logEvent -> logEvent.getMessage().toString().contains("Failed to create reader on __change_events topic")); assertFalse(logFound); boolean logFound2 = testLogAppender.getEvents().stream().anyMatch(logEvent -> logEvent.getMessage().toString().contains("Failed to check the move events for the system topic")); assertTrue(logFound2); - verify(spyService, times(1)).cleanPoliciesCacheInitMap(any(), anyBoolean()); - verify(spyService, times(1)).cleanupFailedPolicyCacheInit(any(), any(), anyBoolean()); + verify(spyService, times(0)).cleanPoliciesCacheInitMap(any()); + verify(spyService, times(2)).cleanupFailedPolicyCacheInit(any(), any(), anyBoolean()); // make sure not occur Recursive update boolean logFound3 = testLogAppender.getEvents().stream().anyMatch(logEvent -> @@ -684,7 +686,7 @@ public void testPrepareInitPoliciesCacheAsyncThrowExceptionInCreateReader() thro || logEvent.getMessage().toString().contains("Failed to read event from the system topic")); assertFalse(logFound2); verify(spyService, times(1)).cleanupFailedPolicyCacheInit(any(), any(), anyBoolean()); - verify(spyService, times(0)).cleanPoliciesCacheInitMap(any(), anyBoolean()); + verify(spyService, times(0)).cleanPoliciesCacheInitMap(any()); } @Test(timeOut = 60_000) @@ -733,22 +735,16 @@ public void testCleanPoliciesCacheInitMapCompletesPendingInitFuture() { // Dropping the cached init future (e.g. on a namespace-bundle unload) must complete it so the topic loads // awaiting it fail fast and retry, instead of hanging until the broker restarts (issue #25294). - CompletableFuture pendingWithReaderClose = new CompletableFuture<>(); - service.policyCacheInitMap.put(namespace, pendingWithReaderClose); - service.cleanPoliciesCacheInitMap(namespace, true); - assertTrue(pendingWithReaderClose.isCompletedExceptionally()); - assertNull(service.getPoliciesCacheInit(namespace)); - - CompletableFuture pendingWithoutReaderClose = new CompletableFuture<>(); - service.policyCacheInitMap.put(namespace, pendingWithoutReaderClose); - service.cleanPoliciesCacheInitMap(namespace, false); - assertTrue(pendingWithoutReaderClose.isCompletedExceptionally()); + CompletableFuture pendingInitFuture = new CompletableFuture<>(); + service.policyCacheInitMap.put(namespace, pendingInitFuture); + service.cleanPoliciesCacheInitMap(namespace); + assertTrue(pendingInitFuture.isCompletedExceptionally()); assertNull(service.getPoliciesCacheInit(namespace)); // An already-completed init future must not be overwritten/disturbed. CompletableFuture alreadyDone = CompletableFuture.completedFuture(null); service.policyCacheInitMap.put(namespace, alreadyDone); - service.cleanPoliciesCacheInitMap(namespace, true); + service.cleanPoliciesCacheInitMap(namespace); assertFalse(alreadyDone.isCompletedExceptionally()); } @@ -783,4 +779,67 @@ public void testCleanupFailedPolicyCacheInitIsIdentityGuarded() { assertNull(service.getReaderCaches().get(namespace)); Mockito.verify(newerReader, Mockito.times(1)).closeAsync(); } + + @Test + @SuppressWarnings("unchecked") + public void testClosedSupersededReaderDoesNotAbortReloadedInit() throws Exception { + // Reproduces the race behind the flaky AdminApi2Test.testGetInternalStatsWithProperties: a namespace unload + // closes the __change_events reader while a reload (e.g. getTopic right after unload) installs a fresh reader + // and init future for the same namespace. The old reader's close only surfaces later, on the pulsar-client + // executor, as an AlreadyClosedException in readMorePoliciesAsync. That late cleanup must NOT clobber the newer + // generation and abort its init future ("...aborted because the cached state was cleared"), which would fail + // the reloading topic load. + @Cleanup + TestLogAppender testLogAppender = TestLogAppender.create(log); + + pulsar.getTopicPoliciesService().close(); + SystemTopicBasedTopicPoliciesService spyService = + Mockito.spy(new SystemTopicBasedTopicPoliciesService(pulsar)); + FieldUtils.writeField(pulsar, "topicPoliciesService", spyService, true); + + final NamespaceName namespace = NamespaceName.get(NAMESPACE5); + admin.namespaces().createNamespace(NAMESPACE5); + + // A real reader, spied so its background read loop is fully controllable: it reports "no more events" so the + // initialization completes and readMorePoliciesAsync starts, then parks on a read future we complete by hand. + SystemTopicClient.Reader oldReader = + Mockito.spy(spyService.createSystemTopicClient(namespace).get(30, TimeUnit.SECONDS)); + CompletableFuture> parkedRead = new CompletableFuture<>(); + Mockito.doReturn(CompletableFuture.completedFuture(false)).when(oldReader).hasMoreEventsAsync(); + Mockito.doReturn(parkedRead).when(oldReader).readNextAsync(); + Mockito.doReturn(CompletableFuture.completedFuture(oldReader)) + .when(spyService).createSystemTopicClient(namespace); + spyService.getReaderCaches().put(namespace, CompletableFuture.completedFuture(oldReader)); + + // Drive initialization: readMorePoliciesAsync(oldReader, ) is now looping, parked on + // parkedRead, having registered its whenComplete callback. + assertTrue(spyService.prepareInitPoliciesCacheAsync(namespace).get(30, TimeUnit.SECONDS)); + Mockito.verify(oldReader, Mockito.atLeastOnce()).readNextAsync(); + + // Simulate the concurrent unload+reload having already replaced the generation: a fresh reader and a fresh, + // still-pending init future that a reloading topic is awaiting. + SystemTopicClient.Reader reloadReader = Mockito.mock(SystemTopicClient.Reader.class); + Mockito.doReturn(CompletableFuture.completedFuture(null)).when(reloadReader).closeAsync(); + CompletableFuture> reloadReaderFuture = + CompletableFuture.completedFuture(reloadReader); + CompletableFuture reloadInitFuture = new CompletableFuture<>(); + spyService.getReaderCaches().put(namespace, reloadReaderFuture); + spyService.policyCacheInitMap.put(namespace, reloadInitFuture); + + // The old reader finally observes it was closed; this runs readMorePoliciesAsync's AlreadyClosed cleanup + // synchronously on this thread. + parkedRead.completeExceptionally(new PulsarClientException.AlreadyClosedException("reader is already closed")); + + // The cleanup ran (it logged), but being identity-guarded on the init future it left the newer generation + // untouched. Before the fix it cleared readerCaches/policyCacheInitMap by namespace key and aborted the reload. + assertTrue(testLogAppender.getEvents().stream().anyMatch(e -> + e.getMessage().toString().contains("Closing the topic policies reader for"))); + assertFalse("the reload's init future must not be aborted by the superseded reader's late close", + reloadInitFuture.isCompletedExceptionally()); + assertFalse(reloadInitFuture.isDone()); + assertSame("the reload's reader must remain cached", reloadReaderFuture, + spyService.getReaderCaches().get(namespace)); + assertSame(reloadInitFuture, spyService.getPoliciesCacheInit(namespace)); + Mockito.verify(reloadReader, Mockito.never()).closeAsync(); + } } From 1a4598d4fa0ef831fa871f841efc50f06433f2c2 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 3 Jul 2026 02:16:19 +0800 Subject: [PATCH 092/213] [improve][broker] Load topic policies on non-persistent topic load and gate the policy replay (#26134) (cherry picked from commit 0629dc5add92c638aa37a01eacb7933723452a5c) Signed-off-by: Zixuan Liu --- .../pulsar/broker/ServiceConfiguration.java | 9 ++ .../pulsar/broker/service/AbstractTopic.java | 106 ++++++++++++++++-- .../SystemTopicBasedTopicPoliciesService.java | 39 +++++-- .../service/TopicPolicyListenerWrapper.java | 68 +++++++++-- .../nonpersistent/NonPersistentTopic.java | 25 ++--- .../service/persistent/PersistentTopic.java | 57 +--------- .../admin/MetadataStoreTopicPoliciesTest.java | 8 ++ .../broker/admin/TopicPoliciesTest.java | 67 +++++++++++ ...temTopicBasedTopicPoliciesServiceTest.java | 64 +++++++++++ .../TopicPolicyListenerWrapperTest.java | 102 +++++++++++++++-- .../persistent/PersistentTopicTest.java | 7 ++ 11 files changed, 450 insertions(+), 102 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index ce8b52751c0f6..ba2a643ddadc9 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -1756,6 +1756,15 @@ The max allowed delay for delayed delivery (in milliseconds). If the broker rece + "please enable the system topic first.") private boolean topicLevelPoliciesEnabled = true; + @FieldContext( + category = CATEGORY_SERVER, + doc = "When enabled, all registered topic-policy listeners in a namespace are re-notified with the current" + + " topic policies after the namespace's topic-policy cache finishes its initial load. Topics load" + + " and apply their own policies when they are loaded, so this broadcast is normally redundant; it" + + " is only needed for custom plugins that register TopicPolicyListeners and depend on it for" + + " backwards compatibility. Disabled by default.") + private boolean topicPolicyListenerReplayEnabled = false; + @FieldContext( category = CATEGORY_SERVER, doc = """ diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index e036f9e1e2e2d..80991077cbbc1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -49,6 +49,7 @@ import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.concurrent.atomic.LongAdder; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Function; import java.util.function.ToLongFunction; import lombok.Getter; import lombok.Setter; @@ -59,6 +60,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.resourcegroup.ResourceGroup; import org.apache.pulsar.broker.resourcegroup.ResourceGroupDispatchLimiter; import org.apache.pulsar.broker.resourcegroup.ResourceGroupPublishLimiter; @@ -119,6 +121,10 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener { protected final BrokerService brokerService; + // Wraps this topic as a TopicPolicyListener so topic-policy updates received while the initial policy is still + // loading are buffered and applied in order once initTopicPolicy() completes initialization. + protected final TopicPolicyListenerWrapper topicPolicyListener = new TopicPolicyListenerWrapper(this); + // Prefix for replication cursors protected final String replicatorPrefix; @@ -183,7 +189,17 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener { AtomicLongFieldUpdater.newUpdater(AbstractTopic.class, "usageCount"); private volatile long usageCount = 0; - private Map subscriptionPolicies = Collections.emptyMap(); + // Effective per-subscription policies, merged from the local and global topic policies with local precedence. + // Unlike the PolicyHierarchyValue-backed fields, subscriptionPolicies is a plain map. It is kept as the merge of + // the two scopes below (rather than assigned directly) because the local-before-global initialization order + // (see TopicPolicyListenerWrapper) would otherwise let the global map -- empty by default in TopicPolicies -- + // overwrite and clear the local per-subscription policies that were applied just before it. + // subscriptionPolicies is volatile because it is read on dispatch threads (getSubscriptionDispatchRate) while it + // is updated on the policy-update thread. localSubscriptionPolicies/globalSubscriptionPolicies are only ever read + // and written on the (single) policy-update thread, so they don't need to be volatile. + private volatile Map subscriptionPolicies = Collections.emptyMap(); + private Map localSubscriptionPolicies = Collections.emptyMap(); + private Map globalSubscriptionPolicies = Collections.emptyMap(); protected final LongAdder msgOutFromRemovedSubscriptions = new LongAdder(); protected final LongAdder bytesOutFromRemovedSubscriptions = new LongAdder(); @@ -350,12 +366,37 @@ protected void updateTopicPolicy(TopicPolicies data) { topicPolicies.getEntryFilters().updateTopicValue(data.getEntryFilters(), isGlobalPolicies); topicPolicies.getDispatcherPauseOnAckStatePersistentEnabled() .updateTopicValue(data.getDispatcherPauseOnAckStatePersistentEnabled(), isGlobalPolicies); - this.subscriptionPolicies = data.getSubscriptionPolicies(); topicPolicies.getResourceGroupName().updateTopicValue(data.getResourceGroupName()); + // Merge instead of assigning directly: keep the local and global per-subscription policies separately and + // recompute the effective map with local precedence, so applying the (default-empty) global map does not + // clear the local per-subscription policies during the local-before-global initialization. + if (isGlobalPolicies) { + globalSubscriptionPolicies = data.getSubscriptionPolicies(); + } else { + localSubscriptionPolicies = data.getSubscriptionPolicies(); + } + subscriptionPolicies = mergeSubscriptionPolicies(globalSubscriptionPolicies, localSubscriptionPolicies); + updateEntryFilters(); } + // Merges the global and local per-subscription policies with local precedence: a subscription present in the + // local policies keeps its local value; otherwise the global value (if any) is used. + private static Map mergeSubscriptionPolicies( + Map globalSubscriptionPolicies, + Map localSubscriptionPolicies) { + if (globalSubscriptionPolicies.isEmpty()) { + return localSubscriptionPolicies; + } + if (localSubscriptionPolicies.isEmpty()) { + return globalSubscriptionPolicies; + } + Map merged = new HashMap<>(globalSubscriptionPolicies); + merged.putAll(localSubscriptionPolicies); + return merged; + } + private void updateTopicLevelReplicatorDispatchRate(Map policy, DispatchRateImpl defaultDispatchRate) { Map dispatchRateMap = new HashMap<>(); @@ -638,12 +679,7 @@ protected boolean isProducersExceeded(boolean isRemote) { } protected TopicPolicyListener getTopicPolicyListener() { - return this; - } - - protected void registerTopicPolicyListener() { - brokerService.getPulsar().getTopicPoliciesService() - .registerListenerAsync(TopicName.getPartitionedTopicName(topic), getTopicPolicyListener()); + return topicPolicyListener; } protected void unregisterTopicPolicyListener() { @@ -651,6 +687,60 @@ protected void unregisterTopicPolicyListener() { .unregisterListener(TopicName.getPartitionedTopicName(topic), getTopicPolicyListener()); } + /** + * Registers the topic-policy listener and applies the topic's initial policies (global and local) to this topic. + * Shared by {@link org.apache.pulsar.broker.service.persistent.PersistentTopic} and + * {@link org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic} so both load their own policies on + * topic load, which removes the need to broadcast every topic's policy when a namespace's policy cache finishes + * loading (see {@code topicPolicyListenerReplayEnabled}). + * + *

Each call re-initializes the listener wrapper and, whatever the outcome, always completes its initialization + * afterwards, so the wrapper never stays in the buffering phase (dropping updates) even if policy loading fails. + * This makes the method safe to run again (e.g. a future retry); runs are expected to be serialized. + */ + protected CompletableFuture initTopicPolicy() { + final var topicPoliciesService = brokerService.getPulsar().getTopicPoliciesService(); + final var partitionedTopicName = TopicName.getPartitionedTopicName(topic); + + // Begin a fresh initialization phase: updates are buffered until initialization completes below. This resets + // any previous phase so the method can be run again. + topicPolicyListener.startInitialization(); + CompletableFuture initTopicPolicyFuture = + topicPoliciesService.registerListenerAsync(partitionedTopicName, topicPolicyListener) + .thenCompose(registered -> { + if (!registered) { + return CompletableFuture.completedFuture(null); + } + if (ExtensibleLoadManagerImpl.isInternalTopic(topic)) { + // Internal topics don't load topic-level policies + return CompletableFuture.completedFuture(null); + } + // future for fetching global topic policies + CompletableFuture> globalPoliciesFuture = + topicPoliciesService.getTopicPoliciesAsync(partitionedTopicName, + TopicPoliciesService.GetType.GLOBAL_ONLY); + // future for fetching local topic policies + CompletableFuture> localPoliciesFuture = + topicPoliciesService.getTopicPoliciesAsync(partitionedTopicName, + TopicPoliciesService.GetType.LOCAL_ONLY); + return globalPoliciesFuture.thenCombine(localPoliciesFuture, (global, local) -> { + // finally update the topic policies with the latest value or loaded value + return CompletableFuture.runAsync(() -> + topicPolicyListener.completeInitialization(global.orElse(null), + local.orElse(null)), + getPoliciesNotifyThread()); + }).thenCompose(Function.identity()); + }); + // Whatever the outcome -- success, failure, or the listener not being registered -- make sure the wrapper + // leaves the initialization (buffering) phase, so it forwards any buffered value plus all future live updates + // instead of dropping them. This is a no-op when the loaded policies were already applied above. Return the + // whenComplete stage (not initTopicPolicyFuture) so the returned future completes only after this has run, and + // whenComplete's pass-through semantics carry the original success or failure to the caller's initialize(). + return initTopicPolicyFuture.whenCompleteAsync((v, ex) -> { + topicPolicyListener.completeInitializationUnlessAlreadyCompleted(); + }, getPoliciesNotifyThread()); + } + protected boolean isSameAddressProducersExceeded(Producer producer) { if (isSystemTopic() || producer.isRemote()) { return false; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index 3fdfcc7b0ad91..56b064fc4a04c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -871,18 +871,43 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp log.debug("[{}] Reach the end of the system topic.", reader.getSystemTopic().getTopicName()); } - // replay policy message - List> notifyFutures = new ArrayList<>(); - for (Map.Entry entry : policiesCache.entrySet()) { - TopicName topicName = entry.getKey(); - TopicPolicies policies = entry.getValue(); - notifyFutures.add(notifyListenersForTopicAsync(topicName, policies)); + // Optionally re-notify the topic-policy listeners for this namespace with the cached policies. + // Topics load their own policies on load (AbstractTopic#initTopicPolicy), so this replay is only + // needed for custom plugins that register TopicPolicyListeners and rely on the broadcast; it is + // off by default (topicPolicyListenerReplayEnabled). + if (pulsarService.getConfiguration().isTopicPolicyListenerReplayEnabled()) { + NamespaceName namespaceObject = reader.getSystemTopic().getTopicName().getNamespaceObject(); + FutureUtil.completeAfter(future, replayTopicPolicyListeners(namespaceObject)); + } else { + future.complete(null); } - FutureUtil.completeAfter(future, FutureUtil.waitForAll(notifyFutures)); } }); } + /** + * Re-notifies the registered topic-policy listeners for every topic in {@code namespace} with the currently + * cached policies (both local and global). Topics apply their own policies on load, so this is only needed for + * custom plugins that register {@link TopicPolicyListener}s and rely on the broadcast when a namespace's policy + * cache finishes loading (see {@code topicPolicyListenerReplayEnabled}). + */ + @VisibleForTesting + CompletableFuture replayTopicPolicyListeners(NamespaceName namespace) { + List> notifyFutures = new ArrayList<>(); + addNamespacePolicyNotifications(policiesCache, namespace, notifyFutures); + addNamespacePolicyNotifications(globalPoliciesCache, namespace, notifyFutures); + return FutureUtil.waitForAll(notifyFutures); + } + + private void addNamespacePolicyNotifications(Map cache, NamespaceName namespace, + List> notifyFutures) { + for (Map.Entry entry : cache.entrySet()) { + if (Objects.equals(entry.getKey().getNamespaceObject(), namespace)) { + notifyFutures.add(notifyListenersForTopicAsync(entry.getKey(), entry.getValue())); + } + } + } + // Full teardown of a namespace's topic-policies state: removes and closes the reader, the message-handler // tracker, the cached policies and the init future. Used when the whole namespace is unloaded. @VisibleForTesting diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapper.java index c04d5d30e12d0..e65acd7741b62 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapper.java @@ -26,10 +26,15 @@ /** * This TopicPolicyListener is used as a wrapper for the real TopicPolicyListener. - * This prevents a race condition in initialization where the topic policy state can change while the - * topic policy state is being applied to the topic in PersistentTopic#initTopicPolicy() method or in - * NonPersistentTopic#initialize method. The impact of the race conditions is that the topic policy state would - * be left in an inconsistent state until another update arrives. This is a rare corner case, but possible. + * This prevents a race condition in initialization where the topic policy state can change while the topic policy + * state is being applied to the topic in AbstractTopic#initTopicPolicy(). The impact of the race condition is that the + * topic policy state would be left inconsistent until another update arrives. This is a rare corner case, but possible. + * + *

Updates received while initializing are buffered (only the latest per scope is kept) and applied by + * {@link #completeInitialization}; updates received afterwards are forwarded immediately. The wrapper is reusable so + * that AbstractTopic#initTopicPolicy() can be run again -- for example to retry it, which this enables but does not + * implement. {@link #startInitialization()} begins a new buffering phase and initialization completes at most once per + * phase. Concurrent initialization phases are not supported; {@code initTopicPolicy} runs are serialized by the caller. */ @Slf4j public class TopicPolicyListenerWrapper implements TopicPolicyListener { @@ -42,12 +47,29 @@ public class TopicPolicyListenerWrapper implements TopicPolicyListener { private Optional latestGlobalPolicies; private Optional latestLocalPolicies; private boolean initialized; - private final long createdTimestampNanos = System.nanoTime(); + // Timestamp when the current initialization phase started, set by startInitialization(). Used only to warn if the + // phase takes too long (i.e. completeInitialization was never called after policy loading started). + private long initializationStartedNanos; private static final long INITIALIZATION_WARNING_LOG_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(30); private int lastIntervalLogged; public TopicPolicyListenerWrapper(TopicPolicyListener realTopicListener) { this.realTopicListener = realTopicListener; + startInitialization(); + } + + /** + * Starts (or restarts) the initialization phase: {@link #onUpdate} buffers updates (keeping only the latest per + * scope) instead of forwarding them, until {@link #completeInitialization} applies them. Called at the start of + * every {@code initTopicPolicy} run so the method can be re-run cleanly (which is what would let a future change + * retry it; no retry is implemented here). Runs before the listener is registered, so no update can arrive before + * the phase (and its warning timer) has started. + */ + public synchronized void startInitialization() { + initialized = false; + latestGlobalPolicies = null; + latestLocalPolicies = null; + initializationStartedNanos = System.nanoTime(); } @Override @@ -77,21 +99,49 @@ public synchronized void onUpdate(TopicPolicies data) { /** * Complete initialization of the TopicPolicyListenerWrapper and emit the latest policies to the real listener. + * * @param loadedGlobalPolicies the loaded global policies - * @param loadedLocalPolicies the loaded local policies + * @param loadedLocalPolicies the loaded local policies */ public synchronized void completeInitialization(TopicPolicies loadedGlobalPolicies, TopicPolicies loadedLocalPolicies) { + // Idempotent: an initialization phase completes at most once. initTopicPolicy runs a terminal + // completeInitializationUnlessAlreadyCompleted() after applying the loaded policies, so a later call must be a + // no-op and must not re-emit policies. A new phase is started explicitly via startInitialization(). + if (initialized) { + return; + } + // The listener might have received a newer value (or a delete) than the loaded one while the loading // was happening; prefer the latest value received during initialization, falling back to the loaded // value only when nothing was received for that scope. - emitInitialPolicies(latestGlobalPolicies, loadedGlobalPolicies); + // + // Emit the local policy before the global policy. A local topic policy takes precedence over a global one, + // so applying the local value first means that by the time the global value is applied the local override is + // already in place and the merged (local-wins) result is what takes effect. Emitting the global value first + // would briefly apply it on its own and let a global-only setting act before the local policy overrides it -- + // e.g. a compaction subscription being created for a global compaction policy even though the local policy + // disables compaction. This does not fully solve such ordering hazards, but it removes them whenever a local + // policy exists. When no local policy exists nothing is emitted for the local scope (see emitInitialPolicies), + // so this ordering does not change behavior for topics that only have a global policy. emitInitialPolicies(latestLocalPolicies, loadedLocalPolicies); + emitInitialPolicies(latestGlobalPolicies, loadedGlobalPolicies); + latestGlobalPolicies = null; latestLocalPolicies = null; initialized = true; } + /** + * Completes initialization with no loaded policies, unless it has already completed. Used as a safety net at the + * end of {@code initTopicPolicy} so the wrapper always leaves the buffering phase -- even when the listener was not + * registered or policy loading failed -- and therefore stops dropping updates: it emits any buffered value and + * forwards all future live updates. A no-op once initialization has completed (e.g. with loaded policies). + */ + public synchronized void completeInitializationUnlessAlreadyCompleted() { + completeInitialization(null, null); + } + private void emitInitialPolicies(Optional latestReceived, TopicPolicies loaded) { if (latestReceived != null) { // A value (or a delete) was received during initialization; it supersedes the loaded value. @@ -104,12 +154,12 @@ private void emitInitialPolicies(Optional latestReceived, TopicPo // warn if the initialization takes too long and updates have been received // this helps detect issues where completeInitialization didn't get called after loading policies private void maybeLogWarning() { - long durationNanos = System.nanoTime() - createdTimestampNanos; + long durationNanos = System.nanoTime() - initializationStartedNanos; int warningLogIntervalCount = (int) (durationNanos / INITIALIZATION_WARNING_LOG_INTERVAL_NANOS); if (warningLogIntervalCount > lastIntervalLogged) { log.warn("TopicPolicyUpdate buffered. TopicPolicyListenerWrapper initialization phase took too long. " + "completeInitialization should have been called to complete the phase. " - + "topicPolicyListener={} sinceCreationMs={}", + + "topicPolicyListener={} sinceInitializationStartedMs={}", realTopicListener, TimeUnit.NANOSECONDS.toMillis(durationNanos)); lastIntervalLogged = warningLogIntervalCount; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index cd072acb180aa..fa617aea9f557 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -66,7 +66,6 @@ import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.TopicAttributes; import org.apache.pulsar.broker.service.TopicPolicyListener; -import org.apache.pulsar.broker.service.TopicPolicyListenerWrapper; import org.apache.pulsar.broker.service.TransportCnx; import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; import org.apache.pulsar.broker.service.schema.exceptions.NotExistSchemaException; @@ -127,9 +126,6 @@ protected TopicStats initialValue() { TOPIC_ATTRIBUTES_FIELD_UPDATER = AtomicReferenceFieldUpdater.newUpdater( NonPersistentTopic.class, TopicAttributes.class, "topicAttributes"); - // prevents race conditions in topic policy initialization - private final TopicPolicyListenerWrapper topicPolicyListener = new TopicPolicyListenerWrapper(this); - private static class TopicStats { public double averageMsgSize; public double aggMsgRateIn; @@ -156,7 +152,6 @@ public void reset() { public NonPersistentTopic(String topic, BrokerService brokerService) { super(topic, brokerService); this.isFenced = false; - registerTopicPolicyListener(); } private CompletableFuture updateClusterMigrated() { @@ -183,13 +178,15 @@ public CompletableFuture initialize() { updateResourceGroupLimiter(); return updateClusterMigrated(); }, getPoliciesNotifyThread()) - // Complete the topic-policy listener wrapper so buffered and future topic-level policy - // updates are forwarded to this topic. Without this the wrapper stays uninitialized forever - // and all topic-level policy updates are silently dropped. Unlike PersistentTopic, - // non-persistent topics don't load initial topic policies (matching the previous behavior), - // so the loaded values are passed as null. - .thenRunAsync(() -> topicPolicyListener.completeInitialization(null, null), - getPoliciesNotifyThread()); + // Load the topic's initial policies (global and local) and register the policy listener, so a + // non-persistent topic applies its own policies on load, the same as a persistent topic does. + .thenCompose(ignore -> initTopicPolicy()) + // a failure to load the initial topic policies must not fail topic loading. + .exceptionally(ex -> { + log.warn("[{}] Error loading topic policies during initialization. Ignoring the failure.", + topic, ex); + return null; + }); } @Override @@ -1321,8 +1318,4 @@ public TopicAttributes getTopicAttributes() { old -> old != null ? old : new TopicAttributes(TopicName.get(topic))); } - @Override - public TopicPolicyListener getTopicPolicyListener() { - return topicPolicyListener; - } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index afb5933789a66..4414bba118dd5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -60,7 +60,6 @@ import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.function.BiConsumer; import java.util.function.BiFunction; -import java.util.function.Function; import lombok.Getter; import lombok.Value; import org.apache.bookkeeper.client.BKException.BKNoSuchLedgerExistsException; @@ -143,8 +142,6 @@ import org.apache.pulsar.broker.service.TopicEventsListener.EventStage; import org.apache.pulsar.broker.service.TopicEventsListener.TopicEvent; import org.apache.pulsar.broker.service.TopicPoliciesService; -import org.apache.pulsar.broker.service.TopicPolicyListener; -import org.apache.pulsar.broker.service.TopicPolicyListenerWrapper; import org.apache.pulsar.broker.service.TransportCnx; import org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage; import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; @@ -302,9 +299,6 @@ protected TopicStatsHelper initialValue() { @Getter private volatile long lastMaxReadPositionMovedForwardTimestamp = 0; - // prevents race conditions in topic policy initialization - private final TopicPolicyListenerWrapper topicPolicyListener = new TopicPolicyListenerWrapper(this); - @Getter private final ExecutorService orderedExecutor; @@ -542,7 +536,8 @@ public CompletableFuture initialize() { .thenCompose(ignore -> initTopicPolicy()) .thenCompose(ignore -> removeOrphanReplicationCursors()) .exceptionally(ex -> { - log.warn("[{}] Error getting policies {} and isEncryptionRequired will be set to false", + log.warn("[{}] Error loading topic policies during initialization. Ignoring the failure. " + + "isEncryptionRequired will be set to false. {}", topic, ex.getMessage()); isEncryptionRequired = false; return null; @@ -4722,50 +4717,6 @@ private void updateSubscriptionsDispatcherRateLimiter() { }); } - protected CompletableFuture initTopicPolicy() { - final var topicPoliciesService = brokerService.pulsar().getTopicPoliciesService(); - final var partitionedTopicName = TopicName.getPartitionedTopicName(topic); - - return topicPoliciesService.registerListenerAsync(partitionedTopicName, topicPolicyListener) - .thenCompose(registered -> { - if (!registered) { - return CompletableFuture.completedFuture(null); - } - if (ExtensibleLoadManagerImpl.isInternalTopic(topic)) { - // Internal topics don't load topic-level policies, but the listener wrapper must - // still be initialized so any buffered/future updates are forwarded to the topic - // instead of being silently dropped. - return CompletableFuture.runAsync( - () -> topicPolicyListener.completeInitialization(null, null), - getPoliciesNotifyThread()); - } - // future for fetching global topic policies - CompletableFuture> globalPoliciesFuture = - topicPoliciesService.getTopicPoliciesAsync(partitionedTopicName, - TopicPoliciesService.GetType.GLOBAL_ONLY); - // future for fetching local topic policies - CompletableFuture> localPoliciesFuture = - topicPoliciesService.getTopicPoliciesAsync(partitionedTopicName, - TopicPoliciesService.GetType.LOCAL_ONLY); - CompletableFuture initialPoliciesFuture = - globalPoliciesFuture.thenCombine(localPoliciesFuture, (global, local) -> { - // finally update the topic policies with the latest value or loaded value - return CompletableFuture.runAsync(() -> - topicPolicyListener.completeInitialization(global.orElse(null), - local.orElse(null)), - getPoliciesNotifyThread()); - }).thenCompose(Function.identity()); - return initialPoliciesFuture.exceptionallyCompose(ex -> - // The topic load path logs and continues when initial policy loading fails. Make sure the - // already-registered wrapper is not left buffering future live updates forever. - CompletableFuture.runAsync( - () -> topicPolicyListener.completeInitialization(null, null), - getPoliciesNotifyThread()) - .thenCompose(__ -> FutureUtil.failedFuture( - FutureUtil.unwrapCompletionException(ex)))); - }); - } - @VisibleForTesting public MessageDeduplication getMessageDeduplication() { return messageDeduplication; @@ -4934,8 +4885,4 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { return future; } - @Override - public TopicPolicyListener getTopicPolicyListener() { - return topicPolicyListener; - } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/MetadataStoreTopicPoliciesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/MetadataStoreTopicPoliciesTest.java index e7fefa164973a..b6927cd5ee8e5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/MetadataStoreTopicPoliciesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/MetadataStoreTopicPoliciesTest.java @@ -44,6 +44,14 @@ public void testTopicPolicyInitialValueWithNamespaceAlreadyLoaded() throws Excep // Not applicable to MetadataStoreTopicPoliciesService. } + @Test(enabled = false) + @Override + public void testNonPersistentTopicAppliesTopicPolicyOnLoad() throws Exception { + // This test is specific to SystemTopicBasedTopicPoliciesService (casts the service and uses + // getPoliciesCacheInit). The non-persistent load-path behavior itself is backend-agnostic and is + // covered against the default SystemTopicBasedTopicPoliciesService in TopicPoliciesTest. + } + @Test(enabled = false) @Override public void testSystemTopicShouldBeCompacted() throws Exception { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java index 3621c9f21d557..302f060953bca 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java @@ -251,6 +251,73 @@ public void testTopicPolicyInitialValueWithNamespaceAlreadyLoaded() throws Excep assertEquals(topic1.getHierarchyTopicPolicies().getMaxSubscriptionsPerTopic().get(), Integer.valueOf(10)); } + @Test + public void testNonPersistentTopicAppliesTopicPolicyOnLoad() throws Exception { + // Non-persistent topics now load and apply their own topic policies on load (like persistent topics), so a + // freshly loaded non-persistent topic must already reflect its topic-level policy without waiting for a + // namespace-wide broadcast. Before this change a non-persistent topic never applied its policies on load. + TopicName topicName = TopicName.get( + TopicDomain.non_persistent.value(), + NamespaceName.get(myNamespace), + "test-np-" + UUID.randomUUID() + ); + String topic = topicName.toString(); + + SystemTopicBasedTopicPoliciesService policyService = + (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); + + admin.topics().createNonPartitionedTopic(topic); + admin.topicPolicies().setMaxSubscriptionsPerTopicAsync(topic, 10).get(); + + //wait until topic loaded with right policy value. + Awaitility.await().untilAsserted(() -> { + AbstractTopic loaded = (AbstractTopic) pulsar.getBrokerService().getTopic(topic, true).get().get(); + assertEquals(loaded.getHierarchyTopicPolicies().getMaxSubscriptionsPerTopic().get(), Integer.valueOf(10)); + }); + + //unload the topic + pulsar.getNamespaceService().unloadNamespaceBundle(pulsar.getNamespaceService().getBundle(topicName)).get(); + assertFalse(pulsar.getBrokerService().getTopics().containsKey(topic)); + + //re-own the namespace bundle without loading the topic + log.info("lookup={}", admin.lookups().lookupTopic(topic)); + assertTrue(pulsar.getBrokerService().isTopicNsOwnedByBrokerAsync(topicName).join()); + assertFalse(pulsar.getBrokerService().getTopics().containsKey(topic)); + //make sure namespace policy reader is fully started. + Awaitility.await().untilAsserted(() -> + assertTrue(policyService.getPoliciesCacheInit(topicName.getNamespaceObject()).isDone())); + + //load the topic: it must already reflect the topic policy, proving it was applied on load, not via a + //later broadcast. + AbstractTopic loaded = (AbstractTopic) pulsar.getBrokerService().getTopic(topic, true).get().get(); + assertEquals(loaded.getHierarchyTopicPolicies().getMaxSubscriptionsPerTopic().get(), Integer.valueOf(10)); + } + + @Test + public void testGlobalPolicyUpdateDoesNotClearLocalSubscriptionPolicy() throws Exception { + // A global topic policy carries no subscription-level overrides by default (an empty subscriptionPolicies + // map). Because AbstractTopic keeps the local and global per-subscription policies separately and merges them + // with local precedence, applying such a global policy must not clear the local per-subscription dispatch-rate + // policy -- which the previous direct assignment would do under the local-before-global ordering. + final String topic = "persistent://" + myNamespace + "/test-sub-policy-merge-" + UUID.randomUUID(); + final String subName = "sub-1"; + admin.topics().createNonPartitionedTopic(topic); + admin.topics().createSubscription(topic, subName, MessageId.earliest); + + DispatchRate localRate = DispatchRateImpl.builder() + .dispatchThrottlingRateInMsg(100).dispatchThrottlingRateInByte(2048).ratePeriodInSecond(1).build(); + admin.topicPolicies().setSubscriptionDispatchRate(topic, subName, localRate); + + AbstractTopic topicRef = (AbstractTopic) pulsar.getBrokerService().getTopic(topic, false).get().orElseThrow(); + Awaitility.await().untilAsserted(() -> + assertEquals(topicRef.getSubscriptionDispatchRate(subName).getDispatchThrottlingRateInMsg(), 100)); + + // Simulate a global topic-policy update that carries no subscription-level overrides. + topicRef.onUpdate(TopicPolicies.builder().isGlobal(true).build()); + + assertEquals(topicRef.getSubscriptionDispatchRate(subName).getDispatchThrottlingRateInMsg(), 100); + } + @Test public void testSetSizeBasedBacklogQuota() throws Exception { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java index 8ea225ebf6278..03081d6d58e7a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java @@ -39,6 +39,7 @@ import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -47,6 +48,7 @@ import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -55,11 +57,13 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.events.PulsarEvent; import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.policies.data.TopicPolicies; import org.apache.pulsar.utils.TestLogAppender; +import org.assertj.core.api.Assertions; import org.awaitility.Awaitility; import org.mockito.Mockito; import org.testng.Assert; @@ -842,4 +846,64 @@ public void testClosedSupersededReaderDoesNotAbortReloadedInit() throws Exceptio assertSame(reloadInitFuture, spyService.getPoliciesCacheInit(namespace)); Mockito.verify(reloadReader, Mockito.never()).closeAsync(); } + + @Test + public void testReplayTopicPolicyListenersNotifiesOnlyNamespaceScopedLocalAndGlobalPolicies() throws Exception { + SystemTopicBasedTopicPoliciesService service = + (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); + final NamespaceName namespaceA = NamespaceName.get(NAMESPACE1); + final NamespaceName namespaceB = NamespaceName.get(NAMESPACE2); + final TopicName localTopicA = TopicName.get("persistent", namespaceA, "replay-local-a"); + final TopicName globalTopicA = TopicName.get("persistent", namespaceA, "replay-global-a"); + final TopicName topicB = TopicName.get("persistent", namespaceB, "replay-b"); + + // Seed the caches: namespace A has one topic with a cached local policy and another with a cached global + // policy; namespace B has a topic with a cached local policy that must not be replayed for namespace A. + service.policiesCache.put(localTopicA, TopicPolicies.builder().isGlobal(false).build()); + service.globalPoliciesCache.put(globalTopicA, TopicPolicies.builder().isGlobal(true).build()); + service.policiesCache.put(topicB, TopicPolicies.builder().isGlobal(false).build()); + + final Map> received = new ConcurrentHashMap<>(); + for (TopicName topicName : List.of(localTopicA, globalTopicA, topicB)) { + final List updates = new CopyOnWriteArrayList<>(); + received.put(topicName, updates); + service.registerListenerAsync(topicName, updates::add).get(); + } + + service.replayTopicPolicyListeners(namespaceA).get(30, TimeUnit.SECONDS); + + // Only namespace A's topics are notified, once each, and both the local and the global cache are replayed. + Assertions.assertThat(received.get(localTopicA)).hasSize(1); + Assertions.assertThat(received.get(globalTopicA)).hasSize(1); + // Namespace B is left untouched. The pre-fix code iterated the whole cache and replayed every namespace. + Assertions.assertThat(received.get(topicB)).isEmpty(); + } + + @Test + public void testTopicPolicyListenerReplayDisabledByDefault() { + Assertions.assertThat(new ServiceConfiguration().isTopicPolicyListenerReplayEnabled()).isFalse(); + } + + @Test + public void testChangeEventsTopicPolicyLoadDoesNotRecurse() throws Exception { + SystemTopicBasedTopicPoliciesService service = + (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); + final String namespaceStr = "system-topic/change-events-recursion"; + admin.namespaces().createNamespace(namespaceStr); + final NamespaceName namespace = NamespaceName.get(namespaceStr); + final TopicName changeEvents = + TopicName.get("persistent", namespace, SystemTopicNames.NAMESPACE_EVENTS_LOCAL_NAME); + + // The __change_events system topic must not load topic-level policies: that would create a policy-cache + // reader on __change_events while __change_events is still loading -- a recursive, deadlocking dependency. + // isSelf() guards getTopicPoliciesAsync so it returns empty for the __change_events topic without ever + // creating a reader. AbstractTopic#initTopicPolicy (now called for persistent AND non-persistent topics) + // relies on this short-circuit when a __change_events topic itself is loaded. + Assertions.assertThat(service.getTopicPoliciesAsync(changeEvents, TopicPoliciesService.GetType.LOCAL_ONLY) + .get(30, TimeUnit.SECONDS)).isEmpty(); + Assertions.assertThat(service.getTopicPoliciesAsync(changeEvents, TopicPoliciesService.GetType.GLOBAL_ONLY) + .get(30, TimeUnit.SECONDS)).isEmpty(); + // No policy-cache reader was created as a side effect, which is what would recurse. + Assertions.assertThat(service.getReaderCaches()).doesNotContainKey(namespace); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapperTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapperTest.java index 0e7553592741d..9d32342aab5d0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapperTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicPolicyListenerWrapperTest.java @@ -56,15 +56,15 @@ public void shouldBufferUpdatesUntilInitializedThenForwardLive() { assertThat(real.updates).isEmpty(); // On completion, the buffered local value wins over the loaded local value; the loaded global value - // is applied since none was buffered. + // is applied since none was buffered. The local policy is emitted before the global one. TopicPolicies loadedGlobal = globalPolicies(); wrapper.completeInitialization(loadedGlobal, localPolicies()); - assertThat(real.updates).containsExactly(loadedGlobal, bufferedLocal); + assertThat(real.updates).containsExactly(bufferedLocal, loadedGlobal); // After initialization, updates are forwarded immediately. TopicPolicies liveUpdate = localPolicies(); wrapper.onUpdate(liveUpdate); - assertThat(real.updates).containsExactly(loadedGlobal, bufferedLocal, liveUpdate); + assertThat(real.updates).containsExactly(bufferedLocal, loadedGlobal, liveUpdate); } @Test @@ -78,7 +78,7 @@ public void shouldPreferBufferedOverLoadedForBothScopes() { wrapper.onUpdate(bufferedLocal); wrapper.completeInitialization(globalPolicies(), localPolicies()); - assertThat(real.updates).containsExactly(bufferedGlobal, bufferedLocal); + assertThat(real.updates).containsExactly(bufferedLocal, bufferedGlobal); } @Test @@ -86,10 +86,12 @@ public void shouldApplyLoadedWhenNothingBuffered() { RecordingListener real = new RecordingListener(); TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real); + // The local policy is emitted before the global policy, so a local topic policy takes precedence over a + // global one once both have been applied. TopicPolicies loadedGlobal = globalPolicies(); TopicPolicies loadedLocal = localPolicies(); wrapper.completeInitialization(loadedGlobal, loadedLocal); - assertThat(real.updates).containsExactly(loadedGlobal, loadedLocal); + assertThat(real.updates).containsExactly(loadedLocal, loadedGlobal); } @Test @@ -118,7 +120,93 @@ public void shouldApplyLatestScopedUpdateOverEarlierDeleteDuringInitialization() wrapper.onUpdate(newerGlobal); wrapper.completeInitialization(globalPolicies(), localPolicies()); - // Global: the newer update wins; Local: the delete (null) wins over the loaded local value. - assertThat(real.updates).containsExactly(newerGlobal, null); + // Local (emitted first): the delete (null) wins over the loaded local value; Global: the newer update wins. + assertThat(real.updates).containsExactly(null, newerGlobal); + } + + @Test + public void shouldNotEmitLocalScopeWhenNoLocalPolicyExists() { + RecordingListener real = new RecordingListener(); + TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real); + + // With no local policy, only the global policy is emitted; no local onUpdate happens, so the + // local-before-global ordering leaves behavior unchanged for topics that only have a global policy. + TopicPolicies loadedGlobal = globalPolicies(); + wrapper.completeInitialization(loadedGlobal, null); + assertThat(real.updates).containsExactly(loadedGlobal); + } + + @Test + public void shouldIgnoreCompleteInitializationAfterAlreadyCompleted() { + RecordingListener real = new RecordingListener(); + TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real); + wrapper.startInitialization(); + + TopicPolicies loadedLocal = localPolicies(); + wrapper.completeInitialization(null, loadedLocal); + assertThat(real.updates).containsExactly(loadedLocal); + + // Completing again (e.g. from initTopicPolicy's terminal handler) must be a no-op and must not re-emit. + wrapper.completeInitialization(globalPolicies(), localPolicies()); + wrapper.completeInitializationUnlessAlreadyCompleted(); + assertThat(real.updates).containsExactly(loadedLocal); + } + + @Test + public void shouldEmitBufferedValueAndForwardLiveUpdatesWhenCompletedWithoutLoadedPolicies() { + RecordingListener real = new RecordingListener(); + TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real); + wrapper.startInitialization(); + + // A policy update arrives while initializing and is buffered. + TopicPolicies buffered = localPolicies(); + wrapper.onUpdate(buffered); + assertThat(real.updates).isEmpty(); + + // initTopicPolicy's terminal handler completes initialization with no loaded policies -- the path taken after + // a policy-load error or when the listener was not registered. The buffered value is emitted and the wrapper + // leaves the buffering phase. + wrapper.completeInitializationUnlessAlreadyCompleted(); + assertThat(real.updates).containsExactly(buffered); + + // Subsequent live updates now flow through instead of being dropped. + TopicPolicies live = globalPolicies(); + wrapper.onUpdate(live); + assertThat(real.updates).containsExactly(buffered, live); + } + + @Test + public void shouldForwardLiveUpdatesAfterCompletingWithNothingBuffered() { + RecordingListener real = new RecordingListener(); + TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real); + wrapper.startInitialization(); + + // Completed with nothing buffered and no loaded policies (e.g. after a failed load): nothing is emitted, but + // the wrapper still leaves the buffering phase so later live updates are forwarded rather than dropped. + wrapper.completeInitializationUnlessAlreadyCompleted(); + assertThat(real.updates).isEmpty(); + + TopicPolicies live = localPolicies(); + wrapper.onUpdate(live); + assertThat(real.updates).containsExactly(live); + } + + @Test + public void shouldRebufferAndReapplyAfterStartInitializationIsCalledAgain() { + RecordingListener real = new RecordingListener(); + TopicPolicyListenerWrapper wrapper = new TopicPolicyListenerWrapper(real); + wrapper.startInitialization(); + TopicPolicies firstLocal = localPolicies(); + wrapper.completeInitialization(null, firstLocal); + assertThat(real.updates).containsExactly(firstLocal); + + // A new initialization phase (e.g. re-running initTopicPolicy): updates are buffered again until it completes, + // and a value buffered during the phase is applied on completion. + wrapper.startInitialization(); + TopicPolicies bufferedGlobal = globalPolicies(); + wrapper.onUpdate(bufferedGlobal); + assertThat(real.updates).containsExactly(firstLocal); + wrapper.completeInitialization(null, null); + assertThat(real.updates).containsExactly(firstLocal, bufferedGlobal); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java index 7717595b2732b..9322946f00e27 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java @@ -743,6 +743,13 @@ class RecordingPersistentTopic extends PersistentTopic { public void onUpdate(TopicPolicies policies) { receivedUpdates.add(policies); } + + // initTopicPolicy() moved to AbstractTopic (a different package), so widen it to public here to keep + // this same-package test able to invoke it directly. + @Override + public CompletableFuture initTopicPolicy() { + return super.initTopicPolicy(); + } } final String topic = "persistent://prop/ns-abc/testTopicPolicyInitFailure-" + UUID.randomUUID(); From 464313959d63635904a3736cb9f99ce527395a66 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Jul 2026 21:51:07 +0300 Subject: [PATCH 093/213] [improve][fn] Upgrade pulsar-client-python to 3.13.0 (#26139) (cherry picked from commit 1b3e9ecb75aceda9053bc3153089284796fd0d74) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 111d31c130516..fab401c628997 100644 --- a/pom.xml +++ b/pom.xml @@ -82,7 +82,7 @@ flexible messaging model and an intuitive client API. ${maven.compiler.target} 8 - 3.12.0 + 3.13.0 21 From ab3aa71630b0a9e07fdfaa314c6fae6c9e8512a4 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 2 Jul 2026 22:37:01 +0300 Subject: [PATCH 094/213] [fix][sec] Upgrade pulsar-client-go to v0.20.0 in pulsar-function-go, also address CVEs (#26140) (cherry picked from commit 82a5cdfb2dc5f6f54ef99e81a7b0242ae61127d9) --- .github/workflows/ci-go-functions.yaml | 4 +- pulsar-function-go/examples/go.mod | 55 +++-- pulsar-function-go/examples/go.sum | 212 +++++++++--------- pulsar-function-go/go.mod | 59 ++--- pulsar-function-go/go.sum | 211 ++++++++--------- pulsar-function-go/pb/generate.sh | 4 +- pulsar-function-go/pf/mockMessage_test.go | 4 + .../latest-version-image/pom.xml | 2 +- 8 files changed, 287 insertions(+), 264 deletions(-) diff --git a/.github/workflows/ci-go-functions.yaml b/.github/workflows/ci-go-functions.yaml index 44dc55f567073..c59291036388b 100644 --- a/.github/workflows/ci-go-functions.yaml +++ b/.github/workflows/ci-go-functions.yaml @@ -67,7 +67,7 @@ jobs: runs-on: ubuntu-24.04 strategy: matrix: - go-version: ['1.24'] + go-version: ['1.25'] steps: - name: Check out code into the Go module directory @@ -85,7 +85,7 @@ jobs: - name: InstallTool run: | cd pulsar-function-go - wget -O - -q https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s v2.0.2 + wget -O - -q https://raw.githubusercontent.com/golangci/golangci-lint/main/install.sh| sh -s v2.12.2 ./bin/golangci-lint --version - name: Build diff --git a/pulsar-function-go/examples/go.mod b/pulsar-function-go/examples/go.mod index 23b89e8e1df9d..b52ccca65d7ab 100644 --- a/pulsar-function-go/examples/go.mod +++ b/pulsar-function-go/examples/go.mod @@ -1,38 +1,38 @@ module github.com/apache/pulsar/pulsar-function-go/examples -go 1.24.0 +go 1.25.0 require ( - github.com/apache/pulsar-client-go v0.14.0 + github.com/apache/pulsar-client-go v0.20.0 github.com/apache/pulsar/pulsar-function-go v0.0.0 ) require ( - github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect - github.com/99designs/keyring v1.2.1 // indirect - github.com/AthenZ/athenz v1.10.39 // indirect + github.com/AthenZ/athenz v1.12.13 // indirect github.com/DataDog/zstd v1.5.0 // indirect + github.com/RoaringBitmap/roaring/v2 v2.8.0 // indirect github.com/ardielle/ardielle-go v1.5.2 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.4.0 // indirect + github.com/bits-and-blooms/bitset v1.12.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/danieljoos/wincred v1.1.2 // indirect - github.com/dvsekhvalnov/jose2go v1.7.0 // indirect - github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-jose/go-jose/v4 v4.1.3 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.2 // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect - github.com/hamba/avro/v2 v2.22.2-0.20240625062549-66aad10411d9 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect + github.com/hamba/avro/v2 v2.29.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.9 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/mtibben/percent v0.2.1 // indirect + github.com/mschoch/smat v0.2.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pierrec/lz4 v2.0.5+incompatible // indirect + github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.20.5 // indirect github.com/prometheus/client_model v0.6.1 // indirect @@ -40,17 +40,26 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - go.uber.org/atomic v1.7.0 // indirect - golang.org/x/mod v0.30.0 // indirect - golang.org/x/net v0.48.0 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.uber.org/atomic v1.11.0 // indirect + golang.org/x/mod v0.35.0 // indirect + golang.org/x/net v0.55.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect - golang.org/x/sys v0.39.0 // indirect - golang.org/x/term v0.38.0 // indirect - golang.org/x/text v0.32.0 // indirect + golang.org/x/sys v0.45.0 // indirect + golang.org/x/text v0.37.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect google.golang.org/grpc v1.79.3 // indirect google.golang.org/protobuf v1.36.10 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apimachinery v0.32.3 // indirect + k8s.io/client-go v0.32.3 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/utils v0.0.0-20250321185631-1f6e0b77f77e // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect ) replace github.com/apache/pulsar/pulsar-function-go => ../ diff --git a/pulsar-function-go/examples/go.sum b/pulsar-function-go/examples/go.sum index 0ccabb4edff78..6a0742b8fb83f 100644 --- a/pulsar-function-go/examples/go.sum +++ b/pulsar-function-go/examples/go.sum @@ -1,107 +1,92 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= -github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= -github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4= -github.com/99designs/keyring v1.2.1 h1:tYLp1ULvO7i3fI5vE21ReQuj99QFSs7lGm0xWyJo87o= -github.com/99designs/keyring v1.2.1/go.mod h1:fc+wB5KTk9wQ9sDx0kFXB3A0MaeGHM9AwRStKOQ5vOA= -github.com/AthenZ/athenz v1.10.39 h1:mtwHTF/v62ewY2Z5KWhuZgVXftBej1/Tn80zx4DcawY= -github.com/AthenZ/athenz v1.10.39/go.mod h1:3Tg8HLsiQZp81BJY58JBeU2BR6B/H4/0MQGfCwhHNEA= +github.com/AthenZ/athenz v1.12.13 h1:OhZNqZsoBXNrKBJobeUUEirPDnwt0HRo4kQMIO1UwwQ= +github.com/AthenZ/athenz v1.12.13/go.mod h1:XXDXXgaQzXaBXnJX6x/bH4yF6eon2lkyzQZ0z/dxprE= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo= github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/Microsoft/hcsshim v0.11.5 h1:haEcLNpj9Ka1gd3B3tAEs9CpE0c+1IhoL59w/exYU38= -github.com/Microsoft/hcsshim v0.11.5/go.mod h1:MV8xMfmECjl5HdO7U/3/hFVnkmSBjAjmA09d4bExKcU= -github.com/apache/pulsar-client-go v0.14.0 h1:P7yfAQhQ52OCAu8yVmtdbNQ81vV8bF54S2MLmCPJC9w= -github.com/apache/pulsar-client-go v0.14.0/go.mod h1:PNUE29x9G1EHMvm41Bs2vcqwgv7N8AEjeej+nEVYbX8= +github.com/RoaringBitmap/roaring/v2 v2.8.0 h1:y1rdtixfXvaITKzkfiKvScI0hlBJHe9sfzJp8cgeM7w= +github.com/RoaringBitmap/roaring/v2 v2.8.0/go.mod h1:FiJcsfkGje/nZBZgCu0ZxCPOKD/hVXDS2dXi7/eUFE0= +github.com/apache/pulsar-client-go v0.20.0 h1:r/jacV7Bf7SR2LiIC3d43vHlPOBUi2vw2yUadot7rV0= +github.com/apache/pulsar-client-go v0.20.0/go.mod h1:/Zf8Q8bSSc6ndEJ8V1muIHf6ZWsMrHoQU+98Ww9pOeI= github.com/ardielle/ardielle-go v1.5.2 h1:TilHTpHIQJ27R1Tl/iITBzMwiUGSlVfiVhwDNGM3Zj4= github.com/ardielle/ardielle-go v1.5.2/go.mod h1:I4hy1n795cUhaVt/ojz83SNVCYIGsAFAONtv2Dr7HUI= -github.com/ardielle/ardielle-tools v1.5.4/go.mod h1:oZN+JRMnqGiIhrzkRN9l26Cej9dEx4jeNG6A+AdkShk= -github.com/aws/aws-sdk-go v1.32.6/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bits-and-blooms/bitset v1.4.0 h1:+YZ8ePm+He2pU3dZlIZiOeAKfrBkXi1lSrXJ/Xzgbu8= -github.com/bits-and-blooms/bitset v1.4.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA= +github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/containerd/containerd v1.7.18 h1:jqjZTQNfXGoEaZdW1WwPU0RqSn1Bm2Ay/KJPUuO8nao= -github.com/containerd/containerd v1.7.18/go.mod h1:IYEk9/IO6wAPUz2bCMVUbsfXjzw5UNP5fLz4PsUygQ4= -github.com/containerd/errdefs v0.1.0 h1:m0wCRBiu1WJT/Fr+iOoQHMQS/eP5myQ8lCv4Dz5ZURM= -github.com/containerd/errdefs v0.1.0/go.mod h1:YgWiiHtLmSeBrvpw+UfPijzbLaB77mEG1WwJTDETIV0= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E= -github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= -github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0= -github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dimfeld/httptreemux v5.0.1+incompatible h1:Qj3gVcDNoOthBAqftuD596rm4wg/adLLz5xh5CmpiCA= github.com/dimfeld/httptreemux v5.0.1+incompatible/go.mod h1:rbUlSV+CCpv/SuqUTP/8Bk2O3LyUV436/yaRGkhP6Z0= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY= -github.com/docker/docker v27.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v28.0.0+incompatible h1:Olh0KS820sJ7nPsBKChVhk5pzqcwDR15fumfAd/p9hM= +github.com/docker/docker v28.0.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dvsekhvalnov/jose2go v1.7.0 h1:bnQc8+GMnidJZA8zc6lLEAb4xNrIqHwO+9TzqvtQZPo= -github.com/dvsekhvalnov/jose2go v1.7.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= +github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= -github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= -github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= -github.com/hamba/avro/v2 v2.22.2-0.20240625062549-66aad10411d9 h1:NEoabXt33PDWK4fXryK4e+XX+fSKDmmu9vg3yb9YI2M= -github.com/hamba/avro/v2 v2.22.2-0.20240625062549-66aad10411d9/go.mod h1:fQVdB2mFZBhPW1D5Abej41LMvrErARGrrdjOnKbm5yw= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/jawher/mow.cli v1.0.4/go.mod h1:5hQj2V8g+qYmLUVWqu4Wuja1pI57M83EChYLVZ0sMKk= -github.com/jawher/mow.cli v1.2.0/go.mod h1:y+pcA3jBAdo/GIZx/0rFjw/K2bVEODP9rfZOfaiq8Ko= -github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/hamba/avro/v2 v2.29.0 h1:fkqoWEPxfygZxrkktgSHEpd0j/P7RKTBTDbcEeMdVEY= +github.com/hamba/avro/v2 v2.29.0/go.mod h1:Pk3T+x74uJoJOFmHrdJ8PRdgSEL/kEKteJ31NytCKxI= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= @@ -110,16 +95,16 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= -github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= -github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= +github.com/moby/sys/user v0.3.0 h1:9ni5DlcW5an3SvRSx4MouotOygvzaXbaSrc/wGDFWPo= +github.com/moby/sys/user v0.3.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -129,27 +114,27 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= -github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= +github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= +github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= -github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= +github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= +github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= @@ -170,28 +155,31 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/testcontainers/testcontainers-go v0.32.0 h1:ug1aK08L3gCHdhknlTTwWjPHPS+/alvLJU/DRxTD/ME= -github.com/testcontainers/testcontainers-go v0.32.0/go.mod h1:CRHrzHLQhlXUsa5gXjTOfqIEJcrK5+xMDmBr/WMI88E= +github.com/testcontainers/testcontainers-go v0.35.0 h1:uADsZpTKFAtp8SLK+hMwSaa+X+JiERHtd4sQAFmXeMo= +github.com/testcontainers/testcontainers-go v0.35.0/go.mod h1:oEVBj5zrfJTrgjwONs1SsRbnBtH9OKl+IGl3UMcr2B4= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 h1:CV7UdSGJt/Ao6Gp4CXckLxVRRsRgDHoI8XjbL3PDl8s= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0/go.mod h1:FRmFuRJfag1IZ2dPkHnEoSFVgTVPUd2qf5Vi69hLb8I= go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= @@ -202,42 +190,46 @@ go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2W go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= -golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= -golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= -golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= +golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= -golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= -golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= -golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= -golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= @@ -247,21 +239,29 @@ google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhH google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +k8s.io/apimachinery v0.32.3 h1:JmDuDarhDmA/Li7j3aPrwhpNBA94Nvk5zLeOge9HH1U= +k8s.io/apimachinery v0.32.3/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +k8s.io/client-go v0.32.3 h1:RKPVltzopkSgHS7aS98QdscAgtgah/+zmpAogooIqVU= +k8s.io/client-go v0.32.3/go.mod h1:3v0+3k4IcT9bXTc4V2rt+d2ZPPG700Xy6Oi0Gdl2PaY= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/utils v0.0.0-20250321185631-1f6e0b77f77e h1:KqK5c/ghOm8xkHYhlodbp6i6+r+ChV2vuAuVRdFbLro= +k8s.io/utils v0.0.0-20250321185631-1f6e0b77f77e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/pulsar-function-go/go.mod b/pulsar-function-go/go.mod index 4b981c563bacc..c9c884adf9041 100644 --- a/pulsar-function-go/go.mod +++ b/pulsar-function-go/go.mod @@ -1,9 +1,9 @@ module github.com/apache/pulsar/pulsar-function-go -go 1.24.0 +go 1.25.0 require ( - github.com/apache/pulsar-client-go v0.14.0 + github.com/apache/pulsar-client-go v0.20.0 github.com/golang/protobuf v1.5.4 github.com/prometheus/client_golang v1.20.5 github.com/prometheus/client_model v0.6.1 @@ -15,45 +15,54 @@ require ( ) require ( - github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect - github.com/99designs/keyring v1.2.1 // indirect - github.com/AthenZ/athenz v1.10.39 // indirect + github.com/AthenZ/athenz v1.12.13 // indirect github.com/DataDog/zstd v1.5.0 // indirect + github.com/RoaringBitmap/roaring/v2 v2.8.0 // indirect github.com/ardielle/ardielle-go v1.5.2 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.4.0 // indirect + github.com/bits-and-blooms/bitset v1.12.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/danieljoos/wincred v1.1.2 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dvsekhvalnov/jose2go v1.7.0 // indirect - github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-jose/go-jose/v4 v4.1.3 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.2 // indirect - github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect - github.com/hamba/avro/v2 v2.22.2-0.20240625062549-66aad10411d9 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect + github.com/hamba/avro/v2 v2.29.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.9 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/kr/text v0.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/mtibben/percent v0.2.1 // indirect + github.com/mschoch/smat v0.2.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pierrec/lz4 v2.0.5+incompatible // indirect + github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - go.uber.org/atomic v1.7.0 // indirect - golang.org/x/mod v0.30.0 // indirect - golang.org/x/net v0.48.0 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.uber.org/atomic v1.11.0 // indirect + golang.org/x/mod v0.35.0 // indirect + golang.org/x/net v0.55.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect - golang.org/x/sys v0.39.0 // indirect - golang.org/x/term v0.38.0 // indirect - golang.org/x/text v0.32.0 // indirect + golang.org/x/sys v0.45.0 // indirect + golang.org/x/text v0.37.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apimachinery v0.32.3 // indirect + k8s.io/client-go v0.32.3 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/utils v0.0.0-20250321185631-1f6e0b77f77e // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect ) replace github.com/apache/pulsar/pulsar-function-go/pf => ./pf diff --git a/pulsar-function-go/go.sum b/pulsar-function-go/go.sum index 0ccabb4edff78..ea72923402897 100644 --- a/pulsar-function-go/go.sum +++ b/pulsar-function-go/go.sum @@ -1,104 +1,90 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= -github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= -github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4= -github.com/99designs/keyring v1.2.1 h1:tYLp1ULvO7i3fI5vE21ReQuj99QFSs7lGm0xWyJo87o= -github.com/99designs/keyring v1.2.1/go.mod h1:fc+wB5KTk9wQ9sDx0kFXB3A0MaeGHM9AwRStKOQ5vOA= -github.com/AthenZ/athenz v1.10.39 h1:mtwHTF/v62ewY2Z5KWhuZgVXftBej1/Tn80zx4DcawY= -github.com/AthenZ/athenz v1.10.39/go.mod h1:3Tg8HLsiQZp81BJY58JBeU2BR6B/H4/0MQGfCwhHNEA= +github.com/AthenZ/athenz v1.12.13 h1:OhZNqZsoBXNrKBJobeUUEirPDnwt0HRo4kQMIO1UwwQ= +github.com/AthenZ/athenz v1.12.13/go.mod h1:XXDXXgaQzXaBXnJX6x/bH4yF6eon2lkyzQZ0z/dxprE= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo= github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/Microsoft/hcsshim v0.11.5 h1:haEcLNpj9Ka1gd3B3tAEs9CpE0c+1IhoL59w/exYU38= -github.com/Microsoft/hcsshim v0.11.5/go.mod h1:MV8xMfmECjl5HdO7U/3/hFVnkmSBjAjmA09d4bExKcU= -github.com/apache/pulsar-client-go v0.14.0 h1:P7yfAQhQ52OCAu8yVmtdbNQ81vV8bF54S2MLmCPJC9w= -github.com/apache/pulsar-client-go v0.14.0/go.mod h1:PNUE29x9G1EHMvm41Bs2vcqwgv7N8AEjeej+nEVYbX8= +github.com/RoaringBitmap/roaring/v2 v2.8.0 h1:y1rdtixfXvaITKzkfiKvScI0hlBJHe9sfzJp8cgeM7w= +github.com/RoaringBitmap/roaring/v2 v2.8.0/go.mod h1:FiJcsfkGje/nZBZgCu0ZxCPOKD/hVXDS2dXi7/eUFE0= +github.com/apache/pulsar-client-go v0.20.0 h1:r/jacV7Bf7SR2LiIC3d43vHlPOBUi2vw2yUadot7rV0= +github.com/apache/pulsar-client-go v0.20.0/go.mod h1:/Zf8Q8bSSc6ndEJ8V1muIHf6ZWsMrHoQU+98Ww9pOeI= github.com/ardielle/ardielle-go v1.5.2 h1:TilHTpHIQJ27R1Tl/iITBzMwiUGSlVfiVhwDNGM3Zj4= github.com/ardielle/ardielle-go v1.5.2/go.mod h1:I4hy1n795cUhaVt/ojz83SNVCYIGsAFAONtv2Dr7HUI= -github.com/ardielle/ardielle-tools v1.5.4/go.mod h1:oZN+JRMnqGiIhrzkRN9l26Cej9dEx4jeNG6A+AdkShk= -github.com/aws/aws-sdk-go v1.32.6/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bits-and-blooms/bitset v1.4.0 h1:+YZ8ePm+He2pU3dZlIZiOeAKfrBkXi1lSrXJ/Xzgbu8= -github.com/bits-and-blooms/bitset v1.4.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA= +github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/containerd/containerd v1.7.18 h1:jqjZTQNfXGoEaZdW1WwPU0RqSn1Bm2Ay/KJPUuO8nao= -github.com/containerd/containerd v1.7.18/go.mod h1:IYEk9/IO6wAPUz2bCMVUbsfXjzw5UNP5fLz4PsUygQ4= -github.com/containerd/errdefs v0.1.0 h1:m0wCRBiu1WJT/Fr+iOoQHMQS/eP5myQ8lCv4Dz5ZURM= -github.com/containerd/errdefs v0.1.0/go.mod h1:YgWiiHtLmSeBrvpw+UfPijzbLaB77mEG1WwJTDETIV0= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E= -github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= -github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0= -github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dimfeld/httptreemux v5.0.1+incompatible h1:Qj3gVcDNoOthBAqftuD596rm4wg/adLLz5xh5CmpiCA= github.com/dimfeld/httptreemux v5.0.1+incompatible/go.mod h1:rbUlSV+CCpv/SuqUTP/8Bk2O3LyUV436/yaRGkhP6Z0= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY= -github.com/docker/docker v27.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v28.0.0+incompatible h1:Olh0KS820sJ7nPsBKChVhk5pzqcwDR15fumfAd/p9hM= +github.com/docker/docker v28.0.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dvsekhvalnov/jose2go v1.7.0 h1:bnQc8+GMnidJZA8zc6lLEAb4xNrIqHwO+9TzqvtQZPo= -github.com/dvsekhvalnov/jose2go v1.7.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= +github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= -github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= -github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= -github.com/hamba/avro/v2 v2.22.2-0.20240625062549-66aad10411d9 h1:NEoabXt33PDWK4fXryK4e+XX+fSKDmmu9vg3yb9YI2M= -github.com/hamba/avro/v2 v2.22.2-0.20240625062549-66aad10411d9/go.mod h1:fQVdB2mFZBhPW1D5Abej41LMvrErARGrrdjOnKbm5yw= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/jawher/mow.cli v1.0.4/go.mod h1:5hQj2V8g+qYmLUVWqu4Wuja1pI57M83EChYLVZ0sMKk= -github.com/jawher/mow.cli v1.2.0/go.mod h1:y+pcA3jBAdo/GIZx/0rFjw/K2bVEODP9rfZOfaiq8Ko= -github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/hamba/avro/v2 v2.29.0 h1:fkqoWEPxfygZxrkktgSHEpd0j/P7RKTBTDbcEeMdVEY= +github.com/hamba/avro/v2 v2.29.0/go.mod h1:Pk3T+x74uJoJOFmHrdJ8PRdgSEL/kEKteJ31NytCKxI= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -110,16 +96,16 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= -github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= -github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= +github.com/moby/sys/user v0.3.0 h1:9ni5DlcW5an3SvRSx4MouotOygvzaXbaSrc/wGDFWPo= +github.com/moby/sys/user v0.3.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -129,27 +115,27 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= -github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= +github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= +github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= -github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= +github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= +github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= @@ -170,28 +156,31 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/testcontainers/testcontainers-go v0.32.0 h1:ug1aK08L3gCHdhknlTTwWjPHPS+/alvLJU/DRxTD/ME= -github.com/testcontainers/testcontainers-go v0.32.0/go.mod h1:CRHrzHLQhlXUsa5gXjTOfqIEJcrK5+xMDmBr/WMI88E= +github.com/testcontainers/testcontainers-go v0.35.0 h1:uADsZpTKFAtp8SLK+hMwSaa+X+JiERHtd4sQAFmXeMo= +github.com/testcontainers/testcontainers-go v0.35.0/go.mod h1:oEVBj5zrfJTrgjwONs1SsRbnBtH9OKl+IGl3UMcr2B4= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 h1:CV7UdSGJt/Ao6Gp4CXckLxVRRsRgDHoI8XjbL3PDl8s= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0/go.mod h1:FRmFuRJfag1IZ2dPkHnEoSFVgTVPUd2qf5Vi69hLb8I= go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= @@ -202,42 +191,46 @@ go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2W go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= -golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= -golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= -golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= +golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= -golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= -golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= -golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= -golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= @@ -247,21 +240,29 @@ google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhH google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +k8s.io/apimachinery v0.32.3 h1:JmDuDarhDmA/Li7j3aPrwhpNBA94Nvk5zLeOge9HH1U= +k8s.io/apimachinery v0.32.3/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +k8s.io/client-go v0.32.3 h1:RKPVltzopkSgHS7aS98QdscAgtgah/+zmpAogooIqVU= +k8s.io/client-go v0.32.3/go.mod h1:3v0+3k4IcT9bXTc4V2rt+d2ZPPG700Xy6Oi0Gdl2PaY= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/utils v0.0.0-20250321185631-1f6e0b77f77e h1:KqK5c/ghOm8xkHYhlodbp6i6+r+ChV2vuAuVRdFbLro= +k8s.io/utils v0.0.0-20250321185631-1f6e0b77f77e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/pulsar-function-go/pb/generate.sh b/pulsar-function-go/pb/generate.sh index 06b61079739ab..a149c73a72272 100755 --- a/pulsar-function-go/pb/generate.sh +++ b/pulsar-function-go/pb/generate.sh @@ -138,12 +138,12 @@ EOF chmod +x "${genScript}" -echo "Running protoc in Docker (golang:1.24-alpine)..." +echo "Running protoc in Docker (golang:1.25-alpine)..." docker run --rm \ -v "${protoDefinitions}:/proto:ro" \ -v "${outDir}:/out" \ -v "${genScript}:/generate_pb.sh:ro" \ - golang:1.24-alpine \ + golang:1.25-alpine \ /generate_pb.sh # Revision of the last commit that touched any .proto file in the proto directory diff --git a/pulsar-function-go/pf/mockMessage_test.go b/pulsar-function-go/pf/mockMessage_test.go index d5bfe0961e365..a34c263486347 100644 --- a/pulsar-function-go/pf/mockMessage_test.go +++ b/pulsar-function-go/pf/mockMessage_test.go @@ -48,6 +48,10 @@ func (m *MockMessage) Payload() []byte { return m.payload } +func (m *MockMessage) IsNullValue() bool { + return m.payload == nil +} + func (m *MockMessage) ID() pulsar.MessageID { return m.messageID } diff --git a/tests/docker-images/latest-version-image/pom.xml b/tests/docker-images/latest-version-image/pom.xml index 158ef937374e9..5728a1dd0ac48 100644 --- a/tests/docker-images/latest-version-image/pom.xml +++ b/tests/docker-images/latest-version-image/pom.xml @@ -31,7 +31,7 @@ pom - golang:1.24-alpine + golang:1.25-alpine From c0af19d553827e04e6bdb9e95e3cdf23c5e02dbd Mon Sep 17 00:00:00 2001 From: Dream95 <864197662@qq.com> Date: Fri, 3 Jul 2026 02:48:21 +0800 Subject: [PATCH 095/213] [fix][fn] Reorder Function Worker shutdown to stop scheduler before runtime manager (#26136) Signed-off-by: Dream95 (cherry picked from commit 7a40adad02c11130a90e178e8b3aaed996fcacbb) --- .../functions/worker/PulsarWorkerService.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java index 233c4fdb6951d..0a098821af2a3 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java @@ -608,14 +608,6 @@ public void stop() { } } - if (null != functionRuntimeManager) { - try { - functionRuntimeManager.close(); - } catch (Exception e) { - log.warn("Failed to close function runtime manager", e); - } - } - if (null != clusterServiceCoordinator) { clusterServiceCoordinator.close(); } @@ -628,6 +620,14 @@ public void stop() { schedulerManager.close(); } + if (null != functionRuntimeManager) { + try { + functionRuntimeManager.close(); + } catch (Exception e) { + log.warn("Failed to close function runtime manager", e); + } + } + if (null != leaderService) { try { leaderService.close(); From a0421bd55db74b0862f913ed4230cda9df2753cf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 Jul 2026 00:30:01 +0300 Subject: [PATCH 096/213] [fix][sec] Bump github.com/go-jose/go-jose/v4 from 4.1.3 to 4.1.4 in /pulsar-function-go (#26142) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Lari Hotari (cherry picked from commit 055882bbcaab29a79ab962ba23f36dab31ad3320) --- pulsar-function-go/examples/go.mod | 2 +- pulsar-function-go/examples/go.sum | 4 ++-- pulsar-function-go/go.mod | 2 +- pulsar-function-go/go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pulsar-function-go/examples/go.mod b/pulsar-function-go/examples/go.mod index b52ccca65d7ab..3040d08d08187 100644 --- a/pulsar-function-go/examples/go.mod +++ b/pulsar-function-go/examples/go.mod @@ -17,7 +17,7 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect - github.com/go-jose/go-jose/v4 v4.1.3 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect diff --git a/pulsar-function-go/examples/go.sum b/pulsar-function-go/examples/go.sum index 6a0742b8fb83f..aebbffd2a138e 100644 --- a/pulsar-function-go/examples/go.sum +++ b/pulsar-function-go/examples/go.sum @@ -51,8 +51,8 @@ github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/ github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= -github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= -github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= diff --git a/pulsar-function-go/go.mod b/pulsar-function-go/go.mod index c9c884adf9041..54a618c7e32aa 100644 --- a/pulsar-function-go/go.mod +++ b/pulsar-function-go/go.mod @@ -25,7 +25,7 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect - github.com/go-jose/go-jose/v4 v4.1.3 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect diff --git a/pulsar-function-go/go.sum b/pulsar-function-go/go.sum index ea72923402897..619d0258f8813 100644 --- a/pulsar-function-go/go.sum +++ b/pulsar-function-go/go.sum @@ -51,8 +51,8 @@ github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/ github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= -github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= -github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= From 2a0f18d6d53d0d8d95e6f1d07790e1bb1f88fd37 Mon Sep 17 00:00:00 2001 From: Eugene Polozhenkov Date: Tue, 7 Jul 2026 15:50:12 -0400 Subject: [PATCH 097/213] [fix][metadata] Fix orphaned UR parent nodes not cleaned up with Oxia metadata backend (#26158) Co-authored-by: ievgenpolozhenkov (cherry picked from commit 8d5cac607646ccda183373ad7b31e6ffb7e13836) Assisted-by: Claude Code (Opus 4.8) --- .../metadata/api/MetadataStoreException.java | 8 ++++++++ .../PulsarLedgerUnderreplicationManager.java | 18 ++++++++++++------ .../metadata/impl/oxia/OxiaMetadataStore.java | 2 +- .../LedgerUnderreplicationManagerTest.java | 4 ++-- 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataStoreException.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataStoreException.java index 2e2b13a266650..5ddf9a253d8eb 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataStoreException.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataStoreException.java @@ -156,6 +156,14 @@ public InvalidPathException(String path) { } } + public static class NotEmptyException extends MetadataStoreException { + private static final long serialVersionUID = 1L; + + public NotEmptyException(String path) { + super("Key '" + path + "' has children"); + } + } + public static MetadataStoreException unwrap(Throwable t) { if (t instanceof MetadataStoreException) { return (MetadataStoreException) t; diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java index 18871be28c116..1ed465c5c7da0 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java @@ -70,6 +70,7 @@ import org.apache.pulsar.metadata.api.extended.CreateOption; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.apache.pulsar.metadata.impl.ZKMetadataStore; +import org.apache.pulsar.metadata.impl.oxia.OxiaMetadataStore; import org.apache.zookeeper.KeeperException; @Slf4j @@ -436,7 +437,8 @@ public void markLedgerReplicated(long ledgerId) throws ReplicationException.Unav if (l != null) { store.delete(getUrLedgerPath(ledgerId), Optional.of(l.getLedgerNodeVersion())) .get(BLOCKING_CALL_TIMEOUT, MILLISECONDS); - if (store instanceof ZKMetadataStore) { + if (store instanceof ZKMetadataStore + || store instanceof OxiaMetadataStore) { try { // clean up the hierarchy String[] parts = getUrLedgerPath(ledgerId).split("/"); @@ -454,11 +456,15 @@ public void markLedgerReplicated(long ledgerId) throws ReplicationException.Unav // It's safe to ignore, it simply means another // ledger in the same hierarchy has been marked as // underreplicated. - if (ee.getCause() instanceof MetadataStoreException && ee.getCause().getCause() - instanceof KeeperException.NotEmptyException) { - //do nothing. - } else { - log.warn("Error deleting underrepcalited ledger parent node", ee); + // Oxia raises NotEmptyException directly; ZK wraps + // KeeperException.NotEmptyException inside a MetadataStoreException. + boolean isNotEmpty = + ee.getCause() instanceof MetadataStoreException.NotEmptyException + || (ee.getCause() instanceof MetadataStoreException + && ee.getCause().getCause() + instanceof KeeperException.NotEmptyException); + if (!isNotEmpty) { + log.warn("Error deleting underreplicated ledger parent node", ee); } } } diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/oxia/OxiaMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/oxia/OxiaMetadataStore.java index 407a927bda4dc..c1e65d4eac1e1 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/oxia/OxiaMetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/oxia/OxiaMetadataStore.java @@ -178,7 +178,7 @@ protected CompletableFuture storeDelete(String path, Optional expect children -> { if (!children.isEmpty()) { return CompletableFuture.failedFuture( - new MetadataStoreException("Key '" + path + "' has children")); + new MetadataStoreException.NotEmptyException(path)); } else { Set delOption = expectedVersion diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerUnderreplicationManagerTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerUnderreplicationManagerTest.java index ac73491a81c65..6508e6943eec0 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerUnderreplicationManagerTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerUnderreplicationManagerTest.java @@ -300,8 +300,8 @@ public void testMarkingAsReplicated(String provider, Supplier urlSupplie assertEquals(l, lB.get(), "Should be the ledger I marked"); } - @Test(dataProvider = "zkImpls", timeOut = 10000) - public void testZkMetasStoreMarkReplicatedDeleteEmptyParentNodes(String provider, Supplier urlSupplier) + @Test(dataProvider = "distributedImpl", timeOut = 10000) + public void testMarkReplicatedDeletesEmptyParentNodes(String provider, Supplier urlSupplier) throws Exception { methodSetup(urlSupplier); From 9dade35406399f1f60aa0e4daaf800fbc9709d42 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 9 Jul 2026 15:24:57 +0300 Subject: [PATCH 098/213] [fix][ci][branch-4.0] Skip testMarkReplicatedDeletesEmptyParentNodes for Etcd - #26158 hasn't been implemented for Etcd --- .../bookkeeper/LedgerUnderreplicationManagerTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerUnderreplicationManagerTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerUnderreplicationManagerTest.java index 6508e6943eec0..1c9231c3084fa 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerUnderreplicationManagerTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerUnderreplicationManagerTest.java @@ -62,6 +62,7 @@ import org.apache.pulsar.metadata.api.NotificationType; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.awaitility.Awaitility; +import org.testng.SkipException; import org.testng.annotations.AfterMethod; import org.testng.annotations.Test; @@ -303,6 +304,9 @@ public void testMarkingAsReplicated(String provider, Supplier urlSupplie @Test(dataProvider = "distributedImpl", timeOut = 10000) public void testMarkReplicatedDeletesEmptyParentNodes(String provider, Supplier urlSupplier) throws Exception { + if (provider.equals("Etcd")) { + throw new SkipException("Etcd doesn't support deleting empty parent nodes"); + } methodSetup(urlSupplier); String missingReplica = "localhost:3181"; From 8b075faa247df14cbe102b4e8a30c196532bc609 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 9 Jul 2026 22:49:21 +0300 Subject: [PATCH 099/213] [fix][sec][branch-4.0] Upgrade Netty to 4.1.136.Final (#26170) --- .../server/src/assemble/LICENSE.bin.txt | 54 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 52 +++++++++--------- pom.xml | 2 +- 3 files changed, 54 insertions(+), 54 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 1fe9c7e1d520a..218d03dfc8ec9 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -293,33 +293,33 @@ The Apache Software License, Version 2.0 - org.apache.commons-commons-lang3-3.19.0.jar - org.apache.commons-commons-text-1.15.0.jar * Netty - - io.netty-netty-buffer-4.1.135.Final.jar - - io.netty-netty-codec-4.1.135.Final.jar - - io.netty-netty-codec-dns-4.1.135.Final.jar - - io.netty-netty-codec-http-4.1.135.Final.jar - - io.netty-netty-codec-http2-4.1.135.Final.jar - - io.netty-netty-codec-socks-4.1.135.Final.jar - - io.netty-netty-codec-haproxy-4.1.135.Final.jar - - io.netty-netty-common-4.1.135.Final.jar - - io.netty-netty-handler-4.1.135.Final.jar - - io.netty-netty-handler-proxy-4.1.135.Final.jar - - io.netty-netty-resolver-4.1.135.Final.jar - - io.netty-netty-resolver-dns-4.1.135.Final.jar - - io.netty-netty-resolver-dns-classes-macos-4.1.135.Final.jar - - io.netty-netty-resolver-dns-native-macos-4.1.135.Final-osx-aarch_64.jar - - io.netty-netty-resolver-dns-native-macos-4.1.135.Final-osx-x86_64.jar - - io.netty-netty-transport-4.1.135.Final.jar - - io.netty-netty-transport-classes-epoll-4.1.135.Final.jar - - io.netty-netty-transport-native-epoll-4.1.135.Final-linux-aarch_64.jar - - io.netty-netty-transport-native-epoll-4.1.135.Final-linux-x86_64.jar - - io.netty-netty-transport-native-unix-common-4.1.135.Final.jar - - io.netty-netty-tcnative-boringssl-static-2.0.77.Final.jar - - io.netty-netty-tcnative-boringssl-static-2.0.77.Final-linux-aarch_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.77.Final-linux-x86_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.77.Final-osx-aarch_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.77.Final-osx-x86_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.77.Final-windows-x86_64.jar - - io.netty-netty-tcnative-classes-2.0.77.Final.jar + - io.netty-netty-buffer-4.1.136.Final.jar + - io.netty-netty-codec-4.1.136.Final.jar + - io.netty-netty-codec-dns-4.1.136.Final.jar + - io.netty-netty-codec-http-4.1.136.Final.jar + - io.netty-netty-codec-http2-4.1.136.Final.jar + - io.netty-netty-codec-socks-4.1.136.Final.jar + - io.netty-netty-codec-haproxy-4.1.136.Final.jar + - io.netty-netty-common-4.1.136.Final.jar + - io.netty-netty-handler-4.1.136.Final.jar + - io.netty-netty-handler-proxy-4.1.136.Final.jar + - io.netty-netty-resolver-4.1.136.Final.jar + - io.netty-netty-resolver-dns-4.1.136.Final.jar + - io.netty-netty-resolver-dns-classes-macos-4.1.136.Final.jar + - io.netty-netty-resolver-dns-native-macos-4.1.136.Final-osx-aarch_64.jar + - io.netty-netty-resolver-dns-native-macos-4.1.136.Final-osx-x86_64.jar + - io.netty-netty-transport-4.1.136.Final.jar + - io.netty-netty-transport-classes-epoll-4.1.136.Final.jar + - io.netty-netty-transport-native-epoll-4.1.136.Final-linux-aarch_64.jar + - io.netty-netty-transport-native-epoll-4.1.136.Final-linux-x86_64.jar + - io.netty-netty-transport-native-unix-common-4.1.136.Final.jar + - io.netty-netty-tcnative-boringssl-static-2.0.78.Final.jar + - io.netty-netty-tcnative-boringssl-static-2.0.78.Final-linux-aarch_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.78.Final-linux-x86_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.78.Final-osx-aarch_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.78.Final-osx-x86_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.78.Final-windows-x86_64.jar + - io.netty-netty-tcnative-classes-2.0.78.Final.jar - io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.26.Final.jar - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 5b0ff548854a8..e18cd436e11e9 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -345,35 +345,35 @@ The Apache Software License, Version 2.0 - commons-text-1.15.0.jar - commons-compress-1.28.0.jar * Netty - - netty-buffer-4.1.135.Final.jar - - netty-codec-4.1.135.Final.jar - - netty-codec-dns-4.1.135.Final.jar - - netty-codec-http-4.1.135.Final.jar - - netty-codec-socks-4.1.135.Final.jar - - netty-codec-haproxy-4.1.135.Final.jar - - netty-common-4.1.135.Final.jar - - netty-handler-4.1.135.Final.jar - - netty-handler-proxy-4.1.135.Final.jar - - netty-resolver-4.1.135.Final.jar - - netty-resolver-dns-4.1.135.Final.jar - - netty-transport-4.1.135.Final.jar - - netty-transport-classes-epoll-4.1.135.Final.jar - - netty-transport-native-epoll-4.1.135.Final-linux-aarch_64.jar - - netty-transport-native-epoll-4.1.135.Final-linux-x86_64.jar - - netty-transport-native-unix-common-4.1.135.Final.jar - - netty-tcnative-boringssl-static-2.0.77.Final.jar - - netty-tcnative-boringssl-static-2.0.77.Final-linux-aarch_64.jar - - netty-tcnative-boringssl-static-2.0.77.Final-linux-x86_64.jar - - netty-tcnative-boringssl-static-2.0.77.Final-osx-aarch_64.jar - - netty-tcnative-boringssl-static-2.0.77.Final-osx-x86_64.jar - - netty-tcnative-boringssl-static-2.0.77.Final-windows-x86_64.jar - - netty-tcnative-classes-2.0.77.Final.jar + - netty-buffer-4.1.136.Final.jar + - netty-codec-4.1.136.Final.jar + - netty-codec-dns-4.1.136.Final.jar + - netty-codec-http-4.1.136.Final.jar + - netty-codec-socks-4.1.136.Final.jar + - netty-codec-haproxy-4.1.136.Final.jar + - netty-common-4.1.136.Final.jar + - netty-handler-4.1.136.Final.jar + - netty-handler-proxy-4.1.136.Final.jar + - netty-resolver-4.1.136.Final.jar + - netty-resolver-dns-4.1.136.Final.jar + - netty-transport-4.1.136.Final.jar + - netty-transport-classes-epoll-4.1.136.Final.jar + - netty-transport-native-epoll-4.1.136.Final-linux-aarch_64.jar + - netty-transport-native-epoll-4.1.136.Final-linux-x86_64.jar + - netty-transport-native-unix-common-4.1.136.Final.jar + - netty-tcnative-boringssl-static-2.0.78.Final.jar + - netty-tcnative-boringssl-static-2.0.78.Final-linux-aarch_64.jar + - netty-tcnative-boringssl-static-2.0.78.Final-linux-x86_64.jar + - netty-tcnative-boringssl-static-2.0.78.Final-osx-aarch_64.jar + - netty-tcnative-boringssl-static-2.0.78.Final-osx-x86_64.jar + - netty-tcnative-boringssl-static-2.0.78.Final-windows-x86_64.jar + - netty-tcnative-classes-2.0.78.Final.jar - netty-incubator-transport-classes-io_uring-0.0.26.Final.jar - netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar - netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar - - netty-resolver-dns-classes-macos-4.1.135.Final.jar - - netty-resolver-dns-native-macos-4.1.135.Final-osx-aarch_64.jar - - netty-resolver-dns-native-macos-4.1.135.Final-osx-x86_64.jar + - netty-resolver-dns-classes-macos-4.1.136.Final.jar + - netty-resolver-dns-native-macos-4.1.136.Final-osx-aarch_64.jar + - netty-resolver-dns-native-macos-4.1.136.Final-osx-x86_64.jar * Prometheus client - simpleclient-0.16.0.jar - simpleclient_log4j2-0.16.0.jar diff --git a/pom.xml b/pom.xml index fab401c628997..6342477e5bd0d 100644 --- a/pom.xml +++ b/pom.xml @@ -187,7 +187,7 @@ flexible messaging model and an intuitive client API. 1.1.10.8 4.1.12.1 5.7.1 - 4.1.135.Final + 4.1.136.Final 0.0.26.Final 12.1.10 From 7838e79001961a061b38096857275fc34c8798ff Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Mon, 13 Jul 2026 13:59:27 +0800 Subject: [PATCH 100/213] [improve][monitor][branch-4.0] Upgrade OpenTelemetry libraries (#26165) --- .../server/src/assemble/LICENSE.bin.txt | 61 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 10 +-- pom.xml | 6 +- pulsar-opentelemetry/pom.xml | 2 +- .../opentelemetry/OpenTelemetryService.java | 31 ++++++---- .../OpenTelemetryServiceTest.java | 17 ++++++ 6 files changed, 75 insertions(+), 52 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 218d03dfc8ec9..a3f25efe4e7d0 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -337,12 +337,12 @@ The Apache Software License, Version 2.0 - io.prometheus-simpleclient_tracer_otel-0.16.0.jar - io.prometheus-simpleclient_tracer_otel_agent-0.16.0.jar * Prometheus exporter - - io.prometheus-prometheus-metrics-config-1.3.10.jar - - io.prometheus-prometheus-metrics-exporter-common-1.3.10.jar - - io.prometheus-prometheus-metrics-exporter-httpserver-1.3.10.jar - - io.prometheus-prometheus-metrics-exposition-formats-no-protobuf-1.3.10.jar - - io.prometheus-prometheus-metrics-model-1.3.10.jar - - io.prometheus-prometheus-metrics-exposition-textformats-1.3.10.jar + - io.prometheus-prometheus-metrics-config-1.5.1.jar + - io.prometheus-prometheus-metrics-exporter-common-1.5.1.jar + - io.prometheus-prometheus-metrics-exporter-httpserver-1.5.1.jar + - io.prometheus-prometheus-metrics-exposition-formats-no-protobuf-1.5.1.jar + - io.prometheus-prometheus-metrics-exposition-textformats-1.5.1.jar + - io.prometheus-prometheus-metrics-model-1.5.1.jar * Jakarta Bean Validation API - jakarta.validation-jakarta.validation-api-2.0.2.jar - javax.validation-validation-api-1.1.0.Final.jar @@ -529,31 +529,30 @@ The Apache Software License, Version 2.0 - io.reactivex.rxjava3-rxjava-3.0.1.jar * RoaringBitmap - org.roaringbitmap-RoaringBitmap-1.6.9.jar - * OpenTelemetry - - io.opentelemetry-opentelemetry-api-1.56.0.jar - - io.opentelemetry-opentelemetry-api-incubator-1.56.0-alpha.jar - - io.opentelemetry-opentelemetry-common-1.56.0.jar - - io.opentelemetry-opentelemetry-context-1.56.0.jar - - io.opentelemetry-opentelemetry-exporter-common-1.56.0.jar - - io.opentelemetry-opentelemetry-exporter-otlp-1.56.0.jar - - io.opentelemetry-opentelemetry-exporter-otlp-common-1.56.0.jar - - io.opentelemetry-opentelemetry-exporter-prometheus-1.56.0-alpha.jar - - io.opentelemetry-opentelemetry-exporter-sender-okhttp-1.56.0.jar - - io.opentelemetry-opentelemetry-sdk-1.56.0.jar - - io.opentelemetry-opentelemetry-sdk-common-1.56.0.jar - - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-1.56.0.jar - - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-spi-1.56.0.jar - - io.opentelemetry-opentelemetry-sdk-logs-1.56.0.jar - - io.opentelemetry-opentelemetry-sdk-metrics-1.56.0.jar - - io.opentelemetry-opentelemetry-sdk-trace-1.56.0.jar - - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-2.21.0.jar - - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-incubator-2.21.0-alpha.jar - - io.opentelemetry.instrumentation-opentelemetry-resources-2.21.0-alpha.jar - - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java17-2.21.0-alpha.jar - - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java8-2.21.0-alpha.jar - - io.opentelemetry.semconv-opentelemetry-semconv-1.37.0.jar - - com.google.cloud.opentelemetry-detector-resources-support-0.36.0.jar - - io.opentelemetry.contrib-opentelemetry-gcp-resources-1.48.0-alpha.jar +* OpenTelemetry + - io.opentelemetry-opentelemetry-api-1.62.0.jar + - io.opentelemetry-opentelemetry-api-incubator-1.62.0-alpha.jar + - io.opentelemetry-opentelemetry-common-1.62.0.jar + - io.opentelemetry-opentelemetry-context-1.62.0.jar + - io.opentelemetry-opentelemetry-exporter-common-1.62.0.jar + - io.opentelemetry-opentelemetry-exporter-otlp-1.62.0.jar + - io.opentelemetry-opentelemetry-exporter-otlp-common-1.62.0.jar + - io.opentelemetry-opentelemetry-exporter-prometheus-1.62.0-alpha.jar + - io.opentelemetry-opentelemetry-exporter-sender-okhttp-1.62.0.jar + - io.opentelemetry-opentelemetry-sdk-1.62.0.jar + - io.opentelemetry-opentelemetry-sdk-common-1.62.0.jar + - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-1.62.0.jar + - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-spi-1.62.0.jar + - io.opentelemetry-opentelemetry-sdk-logs-1.62.0.jar + - io.opentelemetry-opentelemetry-sdk-metrics-1.62.0.jar + - io.opentelemetry-opentelemetry-sdk-trace-1.62.0.jar + - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-2.28.1.jar + - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-incubator-2.28.1-alpha.jar + - io.opentelemetry.instrumentation-opentelemetry-resources-2.28.1-alpha.jar + - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-2.28.1-alpha.jar + - io.opentelemetry.semconv-opentelemetry-semconv-1.41.1.jar + - com.google.cloud.opentelemetry-detector-resources-support-0.36.0.jar + - io.opentelemetry.contrib-opentelemetry-gcp-resources-1.48.0-alpha.jar * Spotify completable-futures - com.spotify-completable-futures-0.3.6.jar * JSpecify diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index e18cd436e11e9..8759bc392ba6b 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -385,11 +385,11 @@ The Apache Software License, Version 2.0 - log4j-core-2.25.4.jar - log4j-slf4j2-impl-2.25.4.jar - log4j-web-2.25.4.jar - * OpenTelemetry - - opentelemetry-api-1.56.0.jar - - opentelemetry-api-incubator-1.56.0-alpha.jar - - opentelemetry-common-1.56.0.jar - - opentelemetry-context-1.56.0.jar +* OpenTelemetry + - opentelemetry-api-1.62.0.jar + - opentelemetry-api-incubator-1.62.0-alpha.jar + - opentelemetry-common-1.62.0.jar + - opentelemetry-context-1.62.0.jar * BookKeeper - bookkeeper-common-allocator-4.17.4.0.jar diff --git a/pom.xml b/pom.xml index 6342477e5bd0d..456d0400d7581 100644 --- a/pom.xml +++ b/pom.xml @@ -308,11 +308,11 @@ flexible messaging model and an intuitive client API. the core logic is switched to java implementation of zstd in org.apache.commons:commons-compress --> 1.5.7-3 2.0.6 - 1.56.0 + 1.62.0 ${opentelemetry.version}-alpha - 2.21.0 + 2.28.1 ${opentelemetry.instrumentation.version}-alpha - 1.37.0 + 1.41.1 4.7.7 1.8 0.3.6 diff --git a/pulsar-opentelemetry/pom.xml b/pulsar-opentelemetry/pom.xml index 6af111d88e78e..ad6cf1824cb1c 100644 --- a/pulsar-opentelemetry/pom.xml +++ b/pulsar-opentelemetry/pom.xml @@ -61,7 +61,7 @@ io.opentelemetry.instrumentation - opentelemetry-runtime-telemetry-java17 + opentelemetry-runtime-telemetry diff --git a/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java b/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java index d143b743d3497..084b0ee3737a3 100644 --- a/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java +++ b/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java @@ -22,7 +22,10 @@ import com.google.common.annotations.VisibleForTesting; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.exporter.prometheus.PrometheusHttpServer; -import io.opentelemetry.instrumentation.runtimemetrics.java17.RuntimeMetrics; +import io.opentelemetry.instrumentation.runtimetelemetry.RuntimeTelemetry; +import io.opentelemetry.instrumentation.runtimetelemetry.RuntimeTelemetryBuilder; +import io.opentelemetry.instrumentation.runtimetelemetry.internal.Experimental; +import io.opentelemetry.instrumentation.runtimetelemetry.internal.Internal; import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk; import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdkBuilder; @@ -44,11 +47,12 @@ public class OpenTelemetryService implements Closeable { public static final String OTEL_SDK_DISABLED_KEY = "otel.sdk.disabled"; + static final String OTEL_EXPORTER_PROMETHEUS_HOST_KEY = "otel.exporter.prometheus.host"; static final int MAX_CARDINALITY_LIMIT = 10000; private final AtomicReference openTelemetrySdkReference = new AtomicReference<>(); - private final AtomicReference runtimeMetricsReference = new AtomicReference<>(); + private final AtomicReference runtimeTelemetryReference = new AtomicReference<>(); /** * Instantiates the OpenTelemetry SDK. All attributes are overridden by system properties or environment @@ -76,7 +80,9 @@ public OpenTelemetryService(String clusterName, // Cardinality limit includes the overflow attribute set, so we need to add 1. "otel.java.metrics.cardinality.limit", Integer.toString(MAX_CARDINALITY_LIMIT + 1), // Reduce number of allocations by using reusable data mode. - "otel.java.exporter.memory_mode", MemoryMode.REUSABLE_DATA.name() + "otel.java.exporter.memory_mode", MemoryMode.REUSABLE_DATA.name(), + // Preserve the pre-1.62.0 Prometheus exporter default of binding to all interfaces. + OTEL_EXPORTER_PROMETHEUS_HOST_KEY, "0.0.0.0" )); sdkBuilder.addResourceCustomizer( @@ -119,12 +125,13 @@ public OpenTelemetryService(String clusterName, openTelemetrySdkReference.set(sdkBuilder.build().getOpenTelemetrySdk()); // For a list of exposed metrics, see https://opentelemetry.io/docs/specs/semconv/runtime/jvm-metrics/ - runtimeMetricsReference.set(RuntimeMetrics.builder(openTelemetrySdkReference.get()) - // disable JFR based telemetry and use only JMX telemetry - .disableAllFeatures() - // enable experimental JMX telemetry in addition - .emitExperimentalTelemetry() - .build()); + RuntimeTelemetryBuilder runtimeTelemetryBuilder = + RuntimeTelemetry.builder(openTelemetrySdkReference.get()); + // Disable JFR-based telemetry and rely on JMX-based metrics only. + Internal.setDisableAllJfrFeatures(runtimeTelemetryBuilder, true); + // Emit experimental JMX-based runtime metrics in addition to the stable ones. + Experimental.setEmitExperimentalMetrics(runtimeTelemetryBuilder, true); + runtimeTelemetryReference.set(runtimeTelemetryBuilder.build()); } public OpenTelemetry getOpenTelemetry() { @@ -133,9 +140,9 @@ public OpenTelemetry getOpenTelemetry() { @Override public void close() { - RuntimeMetrics runtimeMetrics = runtimeMetricsReference.getAndSet(null); - if (runtimeMetrics != null) { - runtimeMetrics.close(); + RuntimeTelemetry runtimeTelemetry = runtimeTelemetryReference.getAndSet(null); + if (runtimeTelemetry != null) { + runtimeTelemetry.close(); } OpenTelemetrySdk openTelemetrySdk = openTelemetrySdkReference.getAndSet(null); if (openTelemetrySdk != null) { diff --git a/pulsar-opentelemetry/src/test/java/org/apache/pulsar/opentelemetry/OpenTelemetryServiceTest.java b/pulsar-opentelemetry/src/test/java/org/apache/pulsar/opentelemetry/OpenTelemetryServiceTest.java index e3dd29cd1b23e..059152eb0c2b7 100644 --- a/pulsar-opentelemetry/src/test/java/org/apache/pulsar/opentelemetry/OpenTelemetryServiceTest.java +++ b/pulsar-opentelemetry/src/test/java/org/apache/pulsar/opentelemetry/OpenTelemetryServiceTest.java @@ -33,6 +33,7 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import lombok.Cleanup; import org.apache.commons.lang3.StringUtils; @@ -153,6 +154,22 @@ public void testMetricCardinalityIsSet() { }); } + @Test + public void testPrometheusExporterDefaultsToAllInterfacesHost() { + var capturedHost = new AtomicReference(); + @Cleanup + var ots = OpenTelemetryService.builder() + .builderCustomizer(getBuilderCustomizer(null, + Map.of(OpenTelemetryService.OTEL_SDK_DISABLED_KEY, "false")) + .andThen(builder -> builder.addPropertiesCustomizer(config -> { + capturedHost.set(config.getString(OpenTelemetryService.OTEL_EXPORTER_PROMETHEUS_HOST_KEY)); + return Map.of(); + }))) + .clusterName("openTelemetryServicePrometheusHostTestCluster") + .build(); + assertThat(capturedHost.get()).isEqualTo("0.0.0.0"); + } + @Test public void testLongCounter() { var longCounter = meter.counterBuilder("dummyLongCounter").build(); From fc9df8001bfafd24185d7c75b0b7f7ae69617a54 Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Tue, 14 Jul 2026 10:41:14 +0800 Subject: [PATCH 101/213] [fix][sec][branch-4.0] Upgrade Jackson version to 2.18.9 (#26187) --- .../server/src/assemble/LICENSE.bin.txt | 22 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 22 +++++++++---------- pom.xml | 2 +- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index a3f25efe4e7d0..3b10881ab2f84 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -249,17 +249,17 @@ The Apache Software License, Version 2.0 - info.picocli-picocli-shell-jline3-4.7.7.jar * High Performance Primitive Collections for Java -- com.carrotsearch-hppc-0.9.1.jar * Jackson - - com.fasterxml.jackson.core-jackson-annotations-2.18.8.jar - - com.fasterxml.jackson.core-jackson-core-2.18.8.jar - - com.fasterxml.jackson.core-jackson-databind-2.18.8.jar - - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.18.8.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.18.8.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.18.8.jar - - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.18.8.jar - - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.18.8.jar - - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.18.8.jar - - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.18.8.jar - - com.fasterxml.jackson.module-jackson-module-parameter-names-2.18.8.jar + - com.fasterxml.jackson.core-jackson-annotations-2.18.9.jar + - com.fasterxml.jackson.core-jackson-core-2.18.9.jar + - com.fasterxml.jackson.core-jackson-databind-2.18.9.jar + - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.18.9.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.18.9.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.18.9.jar + - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.18.9.jar + - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.18.9.jar + - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.18.9.jar + - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.18.9.jar + - com.fasterxml.jackson.module-jackson-module-parameter-names-2.18.9.jar * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.9.1.jar * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar * Fastutil -- it.unimi.dsi-fastutil-8.5.16.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 8759bc392ba6b..f664b0971d6d3 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -313,17 +313,17 @@ The Apache Software License, Version 2.0 - picocli-4.7.7.jar - picocli-shell-jline3-4.7.7.jar * Jackson - - jackson-annotations-2.18.8.jar - - jackson-core-2.18.8.jar - - jackson-databind-2.18.8.jar - - jackson-dataformat-yaml-2.18.8.jar - - jackson-jaxrs-base-2.18.8.jar - - jackson-jaxrs-json-provider-2.18.8.jar - - jackson-module-jaxb-annotations-2.18.8.jar - - jackson-module-jsonSchema-2.18.8.jar - - jackson-datatype-jdk8-2.18.8.jar - - jackson-datatype-jsr310-2.18.8.jar - - jackson-module-parameter-names-2.18.8.jar + - jackson-annotations-2.18.9.jar + - jackson-core-2.18.9.jar + - jackson-databind-2.18.9.jar + - jackson-dataformat-yaml-2.18.9.jar + - jackson-jaxrs-base-2.18.9.jar + - jackson-jaxrs-json-provider-2.18.9.jar + - jackson-module-jaxb-annotations-2.18.9.jar + - jackson-module-jsonSchema-2.18.9.jar + - jackson-datatype-jdk8-2.18.9.jar + - jackson-datatype-jsr310-2.18.9.jar + - jackson-module-parameter-names-2.18.9.jar * Caffeine -- caffeine-2.9.1.jar * Conscrypt -- conscrypt-openjdk-uber-2.5.2.jar * Gson diff --git a/pom.xml b/pom.xml index 456d0400d7581..18c04ea6f810c 100644 --- a/pom.xml +++ b/pom.xml @@ -209,7 +209,7 @@ flexible messaging model and an intuitive client API. 2.0.11 2.0.6 2.0.1 - 2.18.8 + 2.18.9 8.5.16 0.10.2 1.6.2 From e23b2a505267cf9c42b310e94e47b02a735785bb Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Wed, 15 Jul 2026 15:53:40 +0800 Subject: [PATCH 102/213] [fix][sec][branch-4.0] Upgrade Hadoop to 3.5.0 (#26195) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 18c04ea6f810c..e3812d3f992a1 100644 --- a/pom.xml +++ b/pom.xml @@ -256,7 +256,7 @@ flexible messaging model and an intuitive client API. 1.15.16.Final 0.11.1 0.28.0 - 3.4.2 + 3.5.0 3.6.2 ${hadoop3.version} 2.6.3-hadoop3 From 5481b291c6a5a32cfef27da4a902ea25cf97d352 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Tue, 21 Jul 2026 10:14:40 +0800 Subject: [PATCH 103/213] [fix][client] Preserve null values in pulsar-admin schema output (#26196) --- .../client/impl/schema/SchemaInfoTest.java | 38 ++++++--- .../client/impl/schema/SchemaUtils.java | 3 + .../client/impl/schema/SchemaUtilsTest.java | 81 +++++++++++++++++++ 3 files changed, 112 insertions(+), 10 deletions(-) create mode 100644 pulsar-common/src/test/java/org/apache/pulsar/client/impl/schema/SchemaUtilsTest.java diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaInfoTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaInfoTest.java index dab683beb9624..52b3a897317b5 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaInfoTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaInfoTest.java @@ -44,6 +44,16 @@ public class SchemaInfoTest { + " \"properties\": {}\n" + "}"; + private static final String INT32_SCHEMA_INFO_WITH_NULL_PROPERTY = "{\n" + + " \"name\": \"INT32\",\n" + + " \"schema\": \"\",\n" + + " \"type\": \"INT32\",\n" + + " \"timestamp\": 0,\n" + + " \"properties\": {\n" + + " \"key\": null\n" + + " }\n" + + "}"; + private static final String UTF8_SCHEMA_INFO = "{\n" + " \"name\": \"String\",\n" + " \"schema\": \"\",\n" @@ -95,21 +105,24 @@ public class SchemaInfoTest { + " \"BLUE\"\n" + " ]\n" + " }\n" - + " ]\n" + + " ],\n" + + " \"default\": null\n" + " },\n" + " {\n" + " \"name\": \"field1\",\n" + " \"type\": [\n" + " \"null\",\n" + " \"string\"\n" - + " ]\n" + + " ],\n" + + " \"default\": null\n" + " },\n" + " {\n" + " \"name\": \"field2\",\n" + " \"type\": [\n" + " \"null\",\n" + " \"string\"\n" - + " ]\n" + + " ],\n" + + " \"default\": null\n" + " },\n" + " {\n" + " \"name\": \"field3\",\n" @@ -129,7 +142,8 @@ public class SchemaInfoTest { + " }\n" + " ]\n" + " }\n" - + " ]\n" + + " ],\n" + + " \"default\": null\n" + " },\n" + " {\n" + " \"name\": \"fieldUnableNull\",\n" @@ -171,21 +185,24 @@ public class SchemaInfoTest { + " \"BLUE\"\n" + " ]\n" + " }\n" - + " ]\n" + + " ],\n" + + " \"default\": null\n" + " },\n" + " {\n" + " \"name\": \"field1\",\n" + " \"type\": [\n" + " \"null\",\n" + " \"string\"\n" - + " ]\n" + + " ],\n" + + " \"default\": null\n" + " },\n" + " {\n" + " \"name\": \"field2\",\n" + " \"type\": [\n" + " \"null\",\n" + " \"string\"\n" - + " ]\n" + + " ],\n" + + " \"default\": null\n" + " },\n" + " {\n" + " \"name\": \"field3\",\n" @@ -205,7 +222,8 @@ public class SchemaInfoTest { + " }\n" + " ]\n" + " }\n" - + " ]\n" + + " ],\n" + + " \"default\": null\n" + " },\n" + " {\n" + " \"name\": \"fieldUnableNull\",\n" @@ -341,8 +359,8 @@ public void testNullPropertyValue() throws JSONException { .properties(map) .build(); - // null key will be skipped by Gson when serializing JSON to String - JSONAssert.assertEquals(si.toString(), INT32_SCHEMA_INFO, JSONCompareMode.NON_EXTENSIBLE); + JSONAssert.assertEquals(si.toString(), INT32_SCHEMA_INFO_WITH_NULL_PROPERTY, + JSONCompareMode.NON_EXTENSIBLE); } } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/client/impl/schema/SchemaUtils.java b/pulsar-common/src/main/java/org/apache/pulsar/client/impl/schema/SchemaUtils.java index dcc73d1d21fbb..82090936cb493 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/client/impl/schema/SchemaUtils.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/client/impl/schema/SchemaUtils.java @@ -202,6 +202,7 @@ public static String getStringSchemaVersion(byte[] schemaVersionBytes) { */ public static String jsonifySchemaInfo(SchemaInfo schemaInfo, boolean prettyPrinting) { GsonBuilder gsonBuilder = new GsonBuilder() + .serializeNulls() .registerTypeHierarchyAdapter(byte[].class, new ByteArrayToStringAdapter(schemaInfo)) .registerTypeHierarchyAdapter(Map.class, SCHEMA_PROPERTIES_SERIALIZER); if (prettyPrinting) { @@ -218,6 +219,7 @@ public static String jsonifySchemaInfo(SchemaInfo schemaInfo, boolean prettyPrin */ public static String jsonifySchemaInfoWithVersion(SchemaInfoWithVersion schemaInfoWithVersion) { GsonBuilder gsonBuilder = new GsonBuilder() + .serializeNulls() .setPrettyPrinting() .registerTypeHierarchyAdapter(SchemaInfo.class, SCHEMAINFO_ADAPTER) .registerTypeHierarchyAdapter(Map.class, SCHEMA_PROPERTIES_SERIALIZER); @@ -327,6 +329,7 @@ public JsonElement serialize(SchemaInfo schemaInfo, */ public static String jsonifyKeyValueSchemaInfo(KeyValue kvSchemaInfo) { GsonBuilder gsonBuilder = new GsonBuilder() + .serializeNulls() .registerTypeHierarchyAdapter(SchemaInfo.class, SCHEMAINFO_ADAPTER) .registerTypeHierarchyAdapter(Map.class, SCHEMA_PROPERTIES_SERIALIZER); return gsonBuilder.create().toJson(kvSchemaInfo); diff --git a/pulsar-common/src/test/java/org/apache/pulsar/client/impl/schema/SchemaUtilsTest.java b/pulsar-common/src/test/java/org/apache/pulsar/client/impl/schema/SchemaUtilsTest.java new file mode 100644 index 0000000000000..33d069c556328 --- /dev/null +++ b/pulsar-common/src/test/java/org/apache/pulsar/client/impl/schema/SchemaUtilsTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl.schema; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import com.fasterxml.jackson.databind.JsonNode; +import java.util.Collections; +import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.common.schema.SchemaInfoWithVersion; +import org.apache.pulsar.common.schema.SchemaType; +import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.testng.annotations.Test; + +public class SchemaUtilsTest { + + @Test + public void testJsonifySchemaInfoPreservesNullDefaults() throws Exception { + SchemaInfo schemaInfo = new SchemaInfoImpl("test-schema", """ + { + "type": "record", + "name": "Root", + "fields": [ + { + "name": "user_info", + "type": { + "type": "record", + "name": "UserInfo", + "fields": [ + {"name": "user_id", "type": "string", "default": ""}, + {"name": "meter_id", "type": ["null", "string"], "default": null} + ] + }, + "default": {"user_id": "", "meter_id": null} + } + ] + } + """.getBytes(UTF_8), SchemaType.AVRO, 0, Collections.emptyMap()); + + assertNullDefaults(SchemaUtils.jsonifySchemaInfo(schemaInfo, true).getBytes(UTF_8)); + + SchemaInfoWithVersion schemaInfoWithVersion = SchemaInfoWithVersion.builder() + .schemaInfo(schemaInfo) + .version(0) + .build(); + assertNullDefaults(SchemaUtils.jsonifySchemaInfoWithVersion(schemaInfoWithVersion).getBytes(UTF_8)); + } + + private static void assertNullDefaults(byte[] json) throws Exception { + JsonNode schema = ObjectMapperFactory.getMapper().reader().readTree(json).at("/schemaInfo/schema"); + if (schema.isMissingNode()) { + schema = ObjectMapperFactory.getMapper().reader().readTree(json).at("/schema"); + } + + JsonNode userInfoField = schema.at("/fields/0"); + assertTrue(userInfoField.get("default").has("meter_id")); + assertTrue(userInfoField.at("/default/meter_id").isNull()); + + JsonNode meterIdField = userInfoField.at("/type/fields/1"); + assertEquals(meterIdField.get("name").asText(), "meter_id"); + assertTrue(meterIdField.has("default")); + assertTrue(meterIdField.get("default").isNull()); + } +} From e629b76c172d2de0360fcd1fb509f18cba507dc1 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Tue, 21 Jul 2026 10:53:43 +0800 Subject: [PATCH 104/213] [fix][broker] Fix `getEstimatedSizeSinceMarkDeletePosition` throw `IllegalArgumentException` (#26184) --- .../mledger/impl/ManagedCursorImpl.java | 25 +++++- .../mledger/impl/ManagedCursorTest.java | 84 +++++++++++++++++++ 2 files changed, 107 insertions(+), 2 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 6da7b5308b368..7a2ceecbd3d07 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -1291,6 +1291,21 @@ public int getNonContiguousDeletedMessagesRangeSerializedSize() { @Override public long getEstimatedSizeSinceMarkDeletePosition() { + Position markDeletePosition = this.markDeletePosition; + Position lastPosition = ledger.getLastPosition(); + if (markDeletePosition == null || markDeletePosition.compareTo(lastPosition) == 0) { + return 0; + } + if (markDeletePosition.compareTo(lastPosition) > 0) { + if (!ledger.ledgerExists(lastPosition.getLedgerId()) + || isMarkDeletePositionOnEmptyCurrentLedger(markDeletePosition)) { + return 0; + } + throw new IllegalArgumentException(String.format( + "Cursor %s mark-delete position %s is ahead of the last position %s for managed ledger %s", + name, markDeletePosition, lastPosition, ledger.getName())); + } + long totalSize = ledger.estimateBacklogFromPosition(markDeletePosition); // Need to subtract size of individual deleted messages @@ -1303,7 +1318,7 @@ public long getEstimatedSizeSinceMarkDeletePosition() { long deletedCount = 0; lock.readLock().lock(); try { - Range backlogRange = Range.openClosed(markDeletePosition, ledger.getLastPosition()); + Range backlogRange = Range.openClosed(markDeletePosition, lastPosition); if (getConfig().isUnackedRangesOpenCacheSetEnabled()) { deletedCount = individualDeletedMessages.cardinality( @@ -1330,7 +1345,7 @@ public long getEstimatedSizeSinceMarkDeletePosition() { } // Estimate size by using average entry size from the backlog range - Range backlogRange = Range.openClosed(markDeletePosition, ledger.getLastPosition()); + Range backlogRange = Range.openClosed(markDeletePosition, lastPosition); long totalEntriesInBacklog = ledger.getNumberOfEntries(backlogRange); if (totalEntriesInBacklog <= deletedCount || totalEntriesInBacklog == 0) { @@ -1356,6 +1371,12 @@ public long getEstimatedSizeSinceMarkDeletePosition() { return adjustedSize; } + private boolean isMarkDeletePositionOnEmptyCurrentLedger(Position markDeletePosition) { + return ledger.currentLedger != null + && markDeletePosition.getLedgerId() == ledger.currentLedger.getId() + && ledger.currentLedgerEntries == 0; + } + private long getNumberOfEntriesInBacklog() { if (markDeletePosition.compareTo(ledger.getLastPosition()) >= 0) { return 0; diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java index 0865d7ba1677f..bb753f7ec8c69 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java @@ -90,6 +90,7 @@ import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.BookKeeper.DigestType; import org.apache.bookkeeper.client.LedgerEntry; +import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.client.PulsarMockBookKeeper; import org.apache.bookkeeper.client.PulsarMockReadHandleInterceptor; import org.apache.bookkeeper.client.api.LedgerEntries; @@ -3798,6 +3799,89 @@ public void testEstimatedUnackedSize() throws Exception { assertEquals(cursor.getEstimatedSizeSinceMarkDeletePosition(), 10 * entryData.length); } + @Test + public void testEstimatedUnackedSizeWhenLatestLedgerIsEmpty() throws Exception { + ManagedLedger ledger = factory.open("test_estimated_unacked_size_empty_latest_ledger"); + ManagedCursor cursor = ledger.openCursor("c1"); + + assertEquals(cursor.getEstimatedSizeSinceMarkDeletePosition(), 0); + } + + @Test + public void testEstimatedUnackedSizeWhenCursorCaughtUpWithLastPosition() throws Exception { + ManagedLedger ledger = factory.open("test_estimated_unacked_size_cursor_caught_up"); + ManagedCursor cursor = ledger.openCursor("c1"); + + Position lastPosition = ledger.addEntry("entry".getBytes(Encoding)); + cursor.markDelete(lastPosition); + + assertEquals(cursor.getEstimatedSizeSinceMarkDeletePosition(), 0); + } + + @Test + public void testEstimatedUnackedSizeWhenCursorAdvancedToEmptyCurrentLedger() { + ManagedLedgerImpl ledger = mock(ManagedLedgerImpl.class); + when(ledger.getName()).thenReturn("test_estimated_unacked_size_empty_current_ledger"); + when(ledger.getConfig()).thenReturn(new ManagedLedgerConfig()); + when(ledger.getLogger()).thenReturn(log); + + long currentLedgerId = 4; + Position lastPosition = PositionFactory.create(3, 9); + Position markDeletePosition = PositionFactory.create(currentLedgerId, -1); + when(ledger.getLastPosition()).thenReturn(lastPosition); + when(ledger.ledgerExists(lastPosition.getLedgerId())).thenReturn(true); + + LedgerHandle currentLedger = mock(LedgerHandle.class); + when(currentLedger.getId()).thenReturn(currentLedgerId); + ledger.currentLedger = currentLedger; + ledger.currentLedgerEntries = 0; + + ManagedCursorImpl cursor = new ManagedCursorImpl(mock(BookKeeper.class), ledger, "c1"); + cursor.markDeletePosition = markDeletePosition; + + assertEquals(cursor.getEstimatedSizeSinceMarkDeletePosition(), 0); + verify(ledger, never()).estimateBacklogFromPosition(any()); + } + + @Test + public void testEstimatedUnackedSizeWhenLastPositionLedgerIsNoLongerInLedgerList() { + ManagedLedgerImpl ledger = mock(ManagedLedgerImpl.class); + when(ledger.getName()).thenReturn("test_estimated_unacked_size_last_position_ledger_removed"); + when(ledger.getConfig()).thenReturn(new ManagedLedgerConfig()); + when(ledger.getLogger()).thenReturn(log); + + Position lastPosition = PositionFactory.create(3, 0); + Position markDeletePosition = PositionFactory.create(4, -1); + when(ledger.getLastPosition()).thenReturn(lastPosition); + when(ledger.ledgerExists(lastPosition.getLedgerId())).thenReturn(false); + + ManagedCursorImpl cursor = new ManagedCursorImpl(mock(BookKeeper.class), ledger, "c1"); + cursor.markDeletePosition = markDeletePosition; + + assertEquals(cursor.getEstimatedSizeSinceMarkDeletePosition(), 0); + } + + @Test + public void testEstimatedUnackedSizeFailsWhenCursorIsUnexpectedlyAheadOfLastPosition() { + ManagedLedgerImpl ledger = mock(ManagedLedgerImpl.class); + when(ledger.getName()).thenReturn("test_estimated_unacked_size_unexpected_position"); + when(ledger.getConfig()).thenReturn(new ManagedLedgerConfig()); + when(ledger.getLogger()).thenReturn(log); + + Position lastPosition = PositionFactory.create(1, 10); + Position markDeletePosition = PositionFactory.create(2, -1); + when(ledger.getLastPosition()).thenReturn(lastPosition); + when(ledger.ledgerExists(lastPosition.getLedgerId())).thenReturn(true); + when(ledger.ledgerExists(markDeletePosition.getLedgerId())).thenReturn(true); + + ManagedCursorImpl cursor = new ManagedCursorImpl(mock(BookKeeper.class), ledger, "c1"); + cursor.markDeletePosition = markDeletePosition; + + IllegalArgumentException exception = Assert.expectThrows(IllegalArgumentException.class, + cursor::getEstimatedSizeSinceMarkDeletePosition); + assertTrue(exception.getMessage().contains("is ahead of the last position")); + } + /** * Test that cursor.getEstimatedSizeSinceMarkDeletePosition() correctly accounts for individual * message deletions (asyncDelete/individual ack). From eeddaf8a1fbc6bc25329cc6e29743a8ae8c1da57 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Wed, 22 Jul 2026 11:44:49 +0800 Subject: [PATCH 105/213] Fix compile issue --- .../client/impl/schema/SchemaUtilsTest.java | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/pulsar-common/src/test/java/org/apache/pulsar/client/impl/schema/SchemaUtilsTest.java b/pulsar-common/src/test/java/org/apache/pulsar/client/impl/schema/SchemaUtilsTest.java index 33d069c556328..ea04f87f297d1 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/client/impl/schema/SchemaUtilsTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/client/impl/schema/SchemaUtilsTest.java @@ -31,28 +31,29 @@ public class SchemaUtilsTest { + private static final String SCHEMA_WITH_NULL_DEFAULTS = "{\n" + + " \"type\": \"record\",\n" + + " \"name\": \"Root\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\": \"user_info\",\n" + + " \"type\": {\n" + + " \"type\": \"record\",\n" + + " \"name\": \"UserInfo\",\n" + + " \"fields\": [\n" + + " {\"name\": \"user_id\", \"type\": \"string\", \"default\": \"\"},\n" + + " {\"name\": \"meter_id\", \"type\": [\"null\", \"string\"], \"default\": null}\n" + + " ]\n" + + " },\n" + + " \"default\": {\"user_id\": \"\", \"meter_id\": null}\n" + + " }\n" + + " ]\n" + + "}"; + @Test public void testJsonifySchemaInfoPreservesNullDefaults() throws Exception { - SchemaInfo schemaInfo = new SchemaInfoImpl("test-schema", """ - { - "type": "record", - "name": "Root", - "fields": [ - { - "name": "user_info", - "type": { - "type": "record", - "name": "UserInfo", - "fields": [ - {"name": "user_id", "type": "string", "default": ""}, - {"name": "meter_id", "type": ["null", "string"], "default": null} - ] - }, - "default": {"user_id": "", "meter_id": null} - } - ] - } - """.getBytes(UTF_8), SchemaType.AVRO, 0, Collections.emptyMap()); + SchemaInfo schemaInfo = new SchemaInfoImpl("test-schema", SCHEMA_WITH_NULL_DEFAULTS.getBytes(UTF_8), + SchemaType.AVRO, 0, Collections.emptyMap()); assertNullDefaults(SchemaUtils.jsonifySchemaInfo(schemaInfo, true).getBytes(UTF_8)); From c5b405e857cbdc15b352f5435b0273aebec6fa3b Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Wed, 22 Jul 2026 15:33:24 +0800 Subject: [PATCH 106/213] Fix compile issue --- .../org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java index bb753f7ec8c69..8f13c41d407b4 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java @@ -3823,7 +3823,6 @@ public void testEstimatedUnackedSizeWhenCursorAdvancedToEmptyCurrentLedger() { ManagedLedgerImpl ledger = mock(ManagedLedgerImpl.class); when(ledger.getName()).thenReturn("test_estimated_unacked_size_empty_current_ledger"); when(ledger.getConfig()).thenReturn(new ManagedLedgerConfig()); - when(ledger.getLogger()).thenReturn(log); long currentLedgerId = 4; Position lastPosition = PositionFactory.create(3, 9); @@ -3848,7 +3847,6 @@ public void testEstimatedUnackedSizeWhenLastPositionLedgerIsNoLongerInLedgerList ManagedLedgerImpl ledger = mock(ManagedLedgerImpl.class); when(ledger.getName()).thenReturn("test_estimated_unacked_size_last_position_ledger_removed"); when(ledger.getConfig()).thenReturn(new ManagedLedgerConfig()); - when(ledger.getLogger()).thenReturn(log); Position lastPosition = PositionFactory.create(3, 0); Position markDeletePosition = PositionFactory.create(4, -1); @@ -3866,7 +3864,6 @@ public void testEstimatedUnackedSizeFailsWhenCursorIsUnexpectedlyAheadOfLastPosi ManagedLedgerImpl ledger = mock(ManagedLedgerImpl.class); when(ledger.getName()).thenReturn("test_estimated_unacked_size_unexpected_position"); when(ledger.getConfig()).thenReturn(new ManagedLedgerConfig()); - when(ledger.getLogger()).thenReturn(log); Position lastPosition = PositionFactory.create(1, 10); Position markDeletePosition = PositionFactory.create(2, -1); From 2b6580f99aebb94ec3bb2ddca89bfbd8e5ea3d15 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Fri, 5 Dec 2025 15:40:50 +0800 Subject: [PATCH 107/213] [fix][client] Fix lookup request semaphore not release problem (#25038) ### Motivation fix bugs ### Bug if lookup request timeout, `pendingLookupRequestSemaphore` will not be released, so fix it ### Matching PR in forked repository PR in forked repository: https://github.com/congbobo184/pulsar/pull/23 (cherry picked from commit 9d8bf601749d465e2394a1a0db96bfe6b70d13a5) --- .../pulsar/client/impl/LookupRetryTest.java | 34 +++++++++++++++++++ .../apache/pulsar/client/impl/ClientCnx.java | 3 +- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/LookupRetryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/LookupRetryTest.java index 7796671307196..f8ebebd53e43a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/LookupRetryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/LookupRetryTest.java @@ -20,9 +20,12 @@ import static org.apache.pulsar.common.protocol.Commands.newLookupErrorResponse; import static org.apache.pulsar.common.protocol.Commands.newPartitionMetadataResponse; +import static org.testng.AssertJUnit.fail; import com.google.common.collect.Sets; import java.util.Queue; +import java.util.Set; import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -34,13 +37,16 @@ import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.common.api.proto.CommandLookupTopic; import org.apache.pulsar.common.api.proto.CommandPartitionedTopicMetadata; import org.apache.pulsar.common.api.proto.ServerError; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.util.FutureUtil; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -51,6 +57,7 @@ public class LookupRetryTest extends MockedPulsarServiceBaseTest { private static final String subscription = "reader-sub"; private final AtomicInteger connectionsCreated = new AtomicInteger(0); private final ConcurrentHashMap> failureMap = new ConcurrentHashMap<>(); + private static final String TEST_TIME_OUT_SEMAPHORE_RELEASE = "testTimeoutReleasePendingLookupRequestSemaphore"; @BeforeClass @Override @@ -110,6 +117,30 @@ public void testGetPartitionedMetadataRetries() throws Exception { } } + @Test + public void testTimeoutReleasePendingLookupRequestSemaphore() throws Exception { + PulsarClientImpl client = (PulsarClientImpl) newClient(); + + LookupService lookup = client.getLookup(); + + CompletableFuture future = + lookup.getPartitionedTopicMetadata(TopicName.get(TEST_TIME_OUT_SEMAPHORE_RELEASE), false); + try { + future.get(); + fail(); + } catch (Exception e) { + Assert.assertTrue(FutureUtil.unwrapCompletionException(e) + instanceof PulsarClientException.TimeoutException); + } + + Set> clientCnxs = client.getCnxPool().getConnections(); + Assert.assertEquals(clientCnxs.size(), 1); + ClientCnx clientCnx = ((CompletableFuture) clientCnxs.toArray()[0]).get(); + Assert.assertEquals(clientCnx.getPendingLookupRequestSemaphore().availablePermits(), + client.conf.getConcurrentLookupRequest()); + client.close(); + } + @Test public void testTimeoutRetriesOnPartitionMetadata() throws Exception { try (PulsarClient client = newClient(); @@ -282,6 +313,9 @@ private Queue errorList(String topicName) { @Override protected void handlePartitionMetadataRequest(CommandPartitionedTopicMetadata partitionMetadata) { TopicName t = TopicName.get(partitionMetadata.getTopic()); + if (t.getLocalName().equals(TEST_TIME_OUT_SEMAPHORE_RELEASE)) { + return; + } LookupError error = errorList(t.getLocalName()).poll(); if (error == LookupError.TOO_MANY) { final long requestId = partitionMetadata.getRequestId(); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index 3982226390699..510ef500adc7d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -927,7 +927,8 @@ public CompletableFuture newLookup(ByteBuf request, long reque if (pendingLookupRequestSemaphore.tryAcquire()) { future.whenComplete((lookupDataResult, throwable) -> { if (throwable instanceof ConnectException - || throwable instanceof PulsarClientException.LookupException) { + || throwable instanceof PulsarClientException.LookupException + || FutureUtil.unwrapCompletionException(throwable) instanceof TimeoutException) { pendingLookupRequestSemaphore.release(); } }); From 7d1ba7fcbc18fc4dec3b2fa7eb689d8dca4df0b1 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 7 Apr 2026 18:27:59 +0300 Subject: [PATCH 108/213] [improve][ci] Replace trivy-action with sandboxed-trivy-action (#25480) (cherry picked from commit 2394bb10a874c65207d1cc68cff0b8e882e0faf8) --- .github/workflows/pulsar-ci.yaml | 39 ++++++++++++++++---------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 73695b80e7d29..20b61ec4dc80e 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -944,25 +944,26 @@ jobs: - name: Check binary licenses run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz && src/check-binary-license.sh ./distribution/shell/target/apache-pulsar-shell-*-bin.tar.gz -# - name: Run Trivy container scan -# id: trivy_scan -# uses: aquasecurity/trivy-action@v0.35.0 -# if: ${{ github.repository == 'apache/pulsar' && github.event_name != 'pull_request' }} -# continue-on-error: true -# with: -# image-ref: "apachepulsar/pulsar:latest" -# scanners: vuln -# severity: CRITICAL,HIGH,MEDIUM,LOW -# limit-severities-for-sarif: true -# format: 'sarif' -# output: 'trivy-results.sarif' -# -# - name: Upload Trivy scan results to GitHub Security tab -# uses: github/codeql-action/upload-sarif@v3 -# if: ${{ steps.trivy_scan.outcome == 'success' && github.repository == 'apache/pulsar' && github.event_name != 'pull_request' }} -# continue-on-error: true -# with: -# sarif_file: 'trivy-results.sarif' + - name: Run Trivy container scan + id: trivy_scan + uses: lhotari/sandboxed-trivy-action@555963036b2012b44c1071508a236e569db28ebb + if: ${{ github.repository == 'apache/pulsar' && github.event_name != 'pull_request' }} + continue-on-error: true + with: + scan-type: 'image' + scan-ref: "apachepulsar/pulsar:latest" + scanners: vuln + severity: CRITICAL,HIGH,MEDIUM,LOW + limit-severities-for-sarif: true + format: 'sarif' + output: 'trivy-results.sarif' + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v4 + if: ${{ steps.trivy_scan.outcome == 'success' && github.repository == 'apache/pulsar' && github.event_name != 'pull_request' }} + continue-on-error: true + with: + sarif_file: 'trivy-results.sarif' - name: Clean up disk space run: | From a7873921b931d81e08275f0f9ce5909230d9d8c7 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 9 Jul 2026 15:01:07 +0300 Subject: [PATCH 109/213] [fix][ci] Upgrade sandboxed-trivy-action to approved sha (#26169) (cherry picked from commit 59d0ac9357ea6578a983817eee233f31c95f5d1c) --- .github/workflows/pulsar-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 20b61ec4dc80e..803ae9993b1a8 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -946,7 +946,7 @@ jobs: - name: Run Trivy container scan id: trivy_scan - uses: lhotari/sandboxed-trivy-action@555963036b2012b44c1071508a236e569db28ebb + uses: lhotari/sandboxed-trivy-action@f01374b6cc3bf7264ab238293e94f6db7ada6dd0 if: ${{ github.repository == 'apache/pulsar' && github.event_name != 'pull_request' }} continue-on-error: true with: From a6a27e8ae8e3a2fbeabf411cf6f3b540a712534b Mon Sep 17 00:00:00 2001 From: Yike Xiao Date: Fri, 3 Jul 2026 21:34:38 +0800 Subject: [PATCH 110/213] [fix][client] Sync ackSet in client with broker to stop acked messages reaching the DLQ (#26135) (cherry picked from commit e8df873ddfde4d8b5a38a49e4b8765c693d65ab9) --- .../auth/MockedPulsarServiceBaseTest.java | 4 ++ .../client/api/DeadLetterTopicTest.java | 71 +++++++++++++++++++ .../pulsar/client/impl/ConsumerImpl.java | 22 +++--- .../impl/MessagePayloadContextImpl.java | 9 ++- .../client/impl/ZeroQueueConsumerImpl.java | 2 +- .../pulsar/client/impl/ConsumerImplTest.java | 57 +++++++++++++++ 6 files changed, 152 insertions(+), 13 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java index 42ff07524e78c..3cb43c43f66f5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java @@ -196,6 +196,10 @@ protected final void internalSetup(ServiceConfiguration serviceConfiguration) th internalSetup(); } + protected PulsarClient newPulsarClient() throws PulsarClientException { + return newPulsarClient(lookupUrl.toString(), 0); + } + protected PulsarClient newPulsarClient(String url, int intervalInSecs) throws PulsarClientException { ClientBuilder clientBuilder = PulsarClient.builder() diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DeadLetterTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DeadLetterTopicTest.java index 001edb4de4ff7..c15d40e6061e5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DeadLetterTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DeadLetterTopicTest.java @@ -1697,5 +1697,76 @@ public void testCheckUnnecessaryGetPartitionedTopicMetadataWhenUseRetryAndDQL() verify(client, times(0)).getPartitionedTopicMetadata(anyString(), anyBoolean(), anyBoolean()); } + @Test + public void testAckedBatchMessageNotSentToDeadLetterTopicOnFinalRedeliveryRound() throws Exception { + final String topic = newTopicName(); + final int maxRedeliveryCount = 3; + final int batchSize = 5; + final String subscriptionName = "my-subscription"; + + Consumer consumer = pulsarClient.newConsumer(Schema.BYTES) + .topic(topic) + .subscriptionName(subscriptionName) + .subscriptionType(SubscriptionType.Shared) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .enableBatchIndexAcknowledgment(true) + .deadLetterPolicy(DeadLetterPolicy.builder().maxRedeliverCount(maxRedeliveryCount).build()) + .ackTimeout(1, TimeUnit.SECONDS) + .receiverQueueSize(100) + .subscribe(); + + @Cleanup + PulsarClient newPulsarClient = newPulsarClient(); + Consumer deadLetterConsumer = newPulsarClient.newConsumer(Schema.BYTES) + .topic(topic + "-" + subscriptionName + "-DLQ") + .subscriptionName(subscriptionName) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + + Producer producer = pulsarClient.newProducer(Schema.BYTES) + .topic(topic) + .enableBatching(true) + .batchingMaxPublishDelay(1, TimeUnit.SECONDS) + .create(); + List> sendFutures = new ArrayList<>(); + for (int i = 0; i < batchSize; i++) { + sendFutures.add(producer.newMessage().value(("message-" + i).getBytes()).sendAsync()); + } + for (CompletableFuture future : sendFutures) { + future.get(); + } + producer.close(); + + // Batch indices 1 and 2 are deliberately left to time out for the first `maxRedeliveryCount` + // rounds, then explicitly acked on the final round (redeliveryCount == maxRedeliveryCount) -- + // the app's last chance to prevent them from being routed to the DLQ. The other 3 messages in + // the batch are acked immediately on the first delivery. + // Expected deliveries: (batchSize - 2) once each, plus indices 1 and 2 redelivered on every + // round from 0 through maxRedeliveryCount inclusive. + final int expectedDeliveries = (batchSize - 2) + 2 * (maxRedeliveryCount + 1); + int received = 0; + while (received < expectedDeliveries) { + Message message = consumer.receive(5, TimeUnit.SECONDS); + assertNotNull(message, "consumer should keep receiving messages until the batch settles"); + received++; + MessageIdAdv messageId = (MessageIdAdv) message.getMessageId(); + int batchIndex = messageId.getBatchIndex(); + int redeliveryCount = message.getRedeliveryCount(); + if ((batchIndex == 1 || batchIndex == 2) && redeliveryCount < maxRedeliveryCount) { + // Let it time out instead of acking. + continue; + } + consumer.acknowledge(message); + } + + // No message should ever be routed to the DLQ, since every message was explicitly acked at or + // before its final allowed redelivery round. + Message deadLetterMessage = deadLetterConsumer.receive(5, TimeUnit.SECONDS); + assertNull(deadLetterMessage, "no message should have been routed to the DLQ, " + + "but received: " + deadLetterMessage); + + deadLetterConsumer.close(); + consumer.close(); + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 64c67dec0718b..87bb936d181cd 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -23,6 +23,7 @@ import static org.apache.pulsar.common.protocol.Commands.hasChecksum; import static org.apache.pulsar.common.protocol.Commands.serializeWithSize; import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; +import static org.apache.pulsar.common.util.SafeCollectionUtils.longArrayToList; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ComparisonChain; import com.google.common.collect.Iterables; @@ -134,7 +135,6 @@ import org.apache.pulsar.common.util.CompletableFutureCancellationHandler; import org.apache.pulsar.common.util.ExceptionHandler; import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.common.util.SafeCollectionUtils; import org.apache.pulsar.common.util.collections.BitSetRecyclable; import org.apache.pulsar.common.util.collections.ConcurrentBitSet; import org.apache.pulsar.common.util.collections.GrowableArrayBlockingQueue; @@ -142,6 +142,7 @@ import org.slf4j.LoggerFactory; public class ConsumerImpl extends ConsumerBase implements ConnectionHandler.Connection { + private static final long[] EMPTY_ACK_SET = new long[0]; private static final int MAX_REDELIVER_UNACKNOWLEDGED = 1000; final long consumerId; @@ -1408,11 +1409,11 @@ protected void processPayloadByProcessor(final BrokerEntryMetadata brokerEntryMe } void messageReceived(CommandMessage cmdMessage, ByteBuf headersAndPayload, ClientCnx cnx) { - List ackSet = Collections.emptyList(); + long[] ackSet = EMPTY_ACK_SET; if (cmdMessage.getAckSetsCount() > 0) { - ackSet = new ArrayList<>(cmdMessage.getAckSetsCount()); + ackSet = new long[cmdMessage.getAckSetsCount()]; for (int i = 0; i < cmdMessage.getAckSetsCount(); i++) { - ackSet.add(cmdMessage.getAckSetAt(i)); + ackSet[i] = cmdMessage.getAckSetAt(i); } } int redeliveryCount = cmdMessage.getRedeliveryCount(); @@ -1482,7 +1483,7 @@ void messageReceived(CommandMessage cmdMessage, ByteBuf headersAndPayload, Clien if (conf.getPayloadProcessor() != null) { // uncompressedPayload is released in this method so we don't need to call release() again processPayloadByProcessor(brokerEntryMetadata, msgMetadata, - uncompressedPayload, msgId, schema, redeliveryCount, ackSet, consumerEpoch); + uncompressedPayload, msgId, schema, redeliveryCount, longArrayToList(ackSet), consumerEpoch); return; } @@ -1760,7 +1761,7 @@ private void interceptAndComplete(final Message message, final CompletableFut } void receiveIndividualMessagesFromBatch(BrokerEntryMetadata brokerEntryMetadata, MessageMetadata msgMetadata, - int redeliveryCount, List ackSet, ByteBuf uncompressedPayload, + int redeliveryCount, long[] ackSet, ByteBuf uncompressedPayload, MessageIdData messageId, ClientCnx cnx, long consumerEpoch) { int batchSize = msgMetadata.getNumMessagesInBatch(); @@ -1774,8 +1775,9 @@ void receiveIndividualMessagesFromBatch(BrokerEntryMetadata brokerEntryMetadata, BitSet ackSetInMessageId = BatchMessageIdImpl.newAckSet(batchSize); BitSetRecyclable ackBitSet = null; - if (ackSet != null && ackSet.size() > 0) { - ackBitSet = BitSetRecyclable.valueOf(SafeCollectionUtils.longListToArray(ackSet)); + if (ackSet != null && ackSet.length > 0) { + ackBitSet = BitSetRecyclable.valueOf(ackSet); + ackSetInMessageId.and(BitSet.valueOf(ackSet)); } SingleMessageMetadata singleMessageMetadata = new SingleMessageMetadata(); @@ -2669,7 +2671,7 @@ public CompletableFuture seekAsync(MessageId messageId) { final ByteBuf seek; if (msgId.getFirstChunkMessageId() != null) { seek = Commands.newSeek(consumerId, requestId, firstChunkMsgId.getLedgerId(), - firstChunkMsgId.getEntryId(), new long[0]); + firstChunkMsgId.getEntryId(), EMPTY_ACK_SET); } else { final long[] ackSetArr; if (MessageIdAdvUtils.isBatch(msgId)) { @@ -2679,7 +2681,7 @@ public CompletableFuture seekAsync(MessageId messageId) { ackSetArr = ackSet.toLongArray(); ackSet.recycle(); } else { - ackSetArr = new long[0]; + ackSetArr = EMPTY_ACK_SET; } seek = Commands.newSeek(consumerId, requestId, msgId.getLedgerId(), msgId.getEntryId(), ackSetArr); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagePayloadContextImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagePayloadContextImpl.java index f4c9aa2707477..23ebf0a92a15d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagePayloadContextImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagePayloadContextImpl.java @@ -75,8 +75,13 @@ public static MessagePayloadContextImpl get(final BrokerEntryMetadata brokerEntr context.consumer = consumer; context.redeliveryCount = redeliveryCount; context.ackSetInMessageId = BatchMessageIdImpl.newAckSet(context.getNumMessages()); - context.ackBitSet = (ackSet != null && ackSet.size() > 0) - ? BitSetRecyclable.valueOf(SafeCollectionUtils.longListToArray(ackSet)) + boolean isAckSetNotEmpty = ackSet != null && ackSet.size() > 0; + long[] ackSetArray = SafeCollectionUtils.longListToArray(ackSet); + if (isAckSetNotEmpty) { + context.ackSetInMessageId.and(BitSet.valueOf(ackSetArray)); + } + context.ackBitSet = isAckSetNotEmpty + ? BitSetRecyclable.valueOf(ackSetArray) : null; return context; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java index cba7af69f8b09..a5ed76f1368b7 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ZeroQueueConsumerImpl.java @@ -193,7 +193,7 @@ protected void tryTriggerListener() { @Override void receiveIndividualMessagesFromBatch(BrokerEntryMetadata brokerEntryMetadata, MessageMetadata msgMetadata, - int redeliveryCount, List ackSet, ByteBuf uncompressedPayload, + int redeliveryCount, long[] ackSet, ByteBuf uncompressedPayload, MessageIdData messageId, ClientCnx cnx, long consumerEpoch) { rejectBatchMessageByClosingConsumer( diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java index c299e2c72702e..cf47d79178555 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.client.impl; +import static org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.any; @@ -31,6 +32,10 @@ import static org.mockito.Mockito.when; import static org.testng.Assert.assertTrue; import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import java.util.Arrays; +import java.util.BitSet; +import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutorService; @@ -38,17 +43,22 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; +import java.util.stream.Collectors; import lombok.Cleanup; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageIdAdv; +import org.apache.pulsar.client.api.MessagePayload; import org.apache.pulsar.client.api.Messages; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; import org.apache.pulsar.client.impl.conf.TopicConsumerConfigurationData; import org.apache.pulsar.client.util.ExecutorProvider; import org.apache.pulsar.client.util.ScheduledExecutorProvider; +import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.util.Backoff; import org.awaitility.Awaitility; import org.testng.Assert; @@ -336,4 +346,51 @@ public void testUpdateAutoScaleReceiverQueueHintRaceWithConcurrentDrain() { "Hint must reflect the post-enqueue state (pipeline had >=1 message); " + "a concurrent drain of the just-enqueued message must not clear it."); } + + @Test(invocationTimeOut = 1000) + public void testGetMessageAtSyncsAckSetInMessageIdWithBrokerAckSet() { + // Regression test for MessagePayloadContextImpl#getMessageAt: the BatchMessageIdImpl handed + // back to the caller carries a shared ackSetInMessageId bitset that must be seeded from the + // broker-reported ackSet, not a fresh "all unacked" bitset. Otherwise indices the broker + // already knows are acked would be reported as still-outstanding in the returned MessageId, + // which is the same root cause that let acked batch messages leak into the DLQ (see + // ConsumerImpl#receiveIndividualMessagesFromBatch and its ackSetInMessageId.and(...) fix). + final int batchSize = 3; + MessageMetadata messageMetadata = new MessageMetadata() + .setProducerName("test-producer") + .setSequenceId(0) + .setPublishTime(System.currentTimeMillis()) + .setNumMessagesInBatch(batchSize); + + // Broker reports index 0 as already acked (bit cleared); indices 1 and 2 are still + // outstanding (bits set). This mirrors the ackSet the broker attaches on redelivery. + BitSet brokerAckSet = new BitSet(batchSize); + brokerAckSet.set(1); + brokerAckSet.set(2); + List ackSet = Arrays.stream(brokerAckSet.toLongArray()).boxed().collect(Collectors.toList()); + + MessageIdImpl messageId = new MessageIdImpl(1L, 2L, -1); + MessagePayloadContextImpl context = MessagePayloadContextImpl.get( + null, messageMetadata, messageId, consumer, 0, ackSet, DEFAULT_CONSUMER_EPOCH); + MessagePayload payload0 = MessagePayloadImpl.create(Unpooled.wrappedBuffer(new byte[]{0})); + MessagePayload payload1 = MessagePayloadImpl.create(Unpooled.wrappedBuffer(new byte[]{1})); + try { + // Index 0 is already acked per the broker, so it must not be redelivered to the app. + Assert.assertNull(context.getMessageAt(0, batchSize, payload0, false, Schema.BYTES)); + + Message message1 = context.getMessageAt(1, batchSize, payload1, false, Schema.BYTES); + Assert.assertNotNull(message1); + + BitSet ackSetInMessageId = ((MessageIdAdv) message1.getMessageId()).getAckSet(); + Assert.assertFalse(ackSetInMessageId.get(0), + "index 0 was already acked by the broker, so the returned MessageId's ackSet " + + "must reflect it as acked, not fall back to the default all-unacked state"); + Assert.assertTrue(ackSetInMessageId.get(1), "index 1 is still outstanding"); + Assert.assertTrue(ackSetInMessageId.get(2), "index 2 is still outstanding"); + } finally { + payload0.release(); + payload1.release(); + context.recycle(); + } + } } From 7ffef17a4c26593011e7d78c339bf14edb9c77c5 Mon Sep 17 00:00:00 2001 From: Dream95 <864197662@qq.com> Date: Fri, 3 Jul 2026 21:35:56 +0800 Subject: [PATCH 111/213] [fix][client] Fix UnAckedMessageRedeliveryTracker to skip cancelled timeouts (#26043) Signed-off-by: Dream95 (cherry picked from commit e85b1f7d3b69fb418afbf467f53ac58ce876d07c) --- .../client/impl/UnAckedMessageRedeliveryTracker.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedMessageRedeliveryTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedMessageRedeliveryTracker.java index 1405e279f8fe5..076616d87d406 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedMessageRedeliveryTracker.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedMessageRedeliveryTracker.java @@ -59,6 +59,10 @@ public UnAckedMessageRedeliveryTracker(PulsarClientImpl client, ConsumerBase timeout = client.timer().newTimeout(new TimerTask() { @Override public void run(Timeout t) throws Exception { + if (t.isCancelled()) { + return; + } + writeLock.lock(); try { HashSet headPartition = redeliveryTimePartitions.removeFirst(); @@ -73,8 +77,11 @@ public void run(Timeout t) throws Exception { redeliveryTimePartitions.addLast(headPartition); triggerRedelivery(consumerBase); } finally { - writeLock.unlock(); - timeout = client.timer().newTimeout(this, tickDurationInMs, TimeUnit.MILLISECONDS); + try { + timeout = client.timer().newTimeout(this, tickDurationInMs, TimeUnit.MILLISECONDS); + } finally { + writeLock.unlock(); + } } } }, this.tickDurationInMs, TimeUnit.MILLISECONDS); From 830db30ae138c334e1041d442915b7f6ea3d8886 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Thu, 9 Jul 2026 23:03:22 +0800 Subject: [PATCH 112/213] [fix][broker] Read subscription properties directly from cursor (#26159) (cherry picked from commit 803ff962018c7e6229dd26af93cd44cfb5320459) Signed-off-by: Zixuan Liu --- .../persistent/PersistentSubscription.java | 20 +++++++------ .../service/persistent/PersistentTopic.java | 24 ++++++++++++--- .../PersistentSubscriptionTest.java | 30 +++++++++++++++++++ 3 files changed, 61 insertions(+), 13 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index 9e5109f347752..081c2cdf49bfc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -52,7 +52,6 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.InvalidCursorPositionException; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.ScanOutcome; -import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.intercept.BrokerInterceptor; @@ -155,6 +154,14 @@ public PersistentSubscription(PersistentTopic topic, String subscriptionName, Ma this(topic, subscriptionName, cursor, replicated, Collections.emptyMap()); } + /** + * Creates a persistent subscription. + * + * @deprecated use {@link #PersistentSubscription(PersistentTopic, String, ManagedCursor, Boolean)} + * instead. The {@code subscriptionProperties} parameter is no longer read; the + * cursor already carries all subscription properties. + */ + @Deprecated public PersistentSubscription(PersistentTopic topic, String subscriptionName, ManagedCursor cursor, Boolean replicated, Map subscriptionProperties) { this.topic = topic; @@ -165,8 +172,6 @@ public PersistentSubscription(PersistentTopic topic, String subscriptionName, Ma this.fullName = MoreObjects.toStringHelper(this).add("topic", topicName).add("name", subName).toString(); this.expiryMonitor = new PersistentMessageExpiryMonitor(topic, subscriptionName, cursor, this); this.setReplicated(replicated); - this.subscriptionProperties = MapUtils.isEmpty(subscriptionProperties) - ? Collections.emptyMap() : Collections.unmodifiableMap(subscriptionProperties); if (config.isTransactionCoordinatorEnabled() && !isEventSystemTopic(TopicName.get(topicName)) && !ExtensibleLoadManagerImpl.isInternalTopic(topicName)) { @@ -1452,7 +1457,7 @@ public CompletableFuture getStatsAsync(GetStatsOptions ge subStats.msgRateExpired = expiryMonitor.getMessageExpiryRate(); subStats.totalMsgExpired = expiryMonitor.getTotalMessageExpired(); subStats.isReplicated = isReplicated(); - subStats.subscriptionProperties = subscriptionProperties; + subStats.subscriptionProperties = getSubscriptionProperties(); subStats.isDurable = cursor.isDurable(); if (getType() == SubType.Key_Shared && dispatcher instanceof StickyKeyDispatcher) { StickyKeyDispatcher keySharedDispatcher = (StickyKeyDispatcher) dispatcher; @@ -1586,7 +1591,7 @@ public boolean isSubscriptionMigrated() { @Override public Map getSubscriptionProperties() { - return subscriptionProperties; + return cursor.getCursorProperties(); } public Position getPositionInPendingAck(Position position) { @@ -1600,10 +1605,7 @@ public CompletableFuture updateSubscriptionProperties(Map } else { newSubscriptionProperties = Collections.unmodifiableMap(subscriptionProperties); } - return cursor.setCursorProperties(newSubscriptionProperties) - .thenRun(() -> { - this.subscriptionProperties = newSubscriptionProperties; - }); + return cursor.setCursorProperties(newSubscriptionProperties); } /** * Return a merged map that contains the cursor properties specified by used diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 4414bba118dd5..b236762ea12f3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1192,6 +1192,7 @@ private CompletableFuture getNonDurableSubscription(Stri synchronized (ledger) { // Create a new non-durable cursor only for the first consumer that connects PersistentSubscription subscription = subscriptions.get(subscriptionName); + CompletableFuture initPropertiesFuture = CompletableFuture.completedFuture(null); if (subscription == null) { MessageIdImpl msgId = startMessageId != null ? (MessageIdImpl) startMessageId @@ -1218,9 +1219,16 @@ private CompletableFuture getNonDurableSubscription(Stri return FutureUtil.failedFuture(e); } - subscription = new PersistentSubscription(this, subscriptionName, cursor, false, - subscriptionProperties); + subscription = new PersistentSubscription(this, subscriptionName, cursor, false); subscriptions.put(subscriptionName, subscription); + + if (subscriptionProperties != null && !subscriptionProperties.isEmpty()) { + // Trade-off: subscriptionProperties should be received by the cursor at creation time, + // the way durable cursors take cursorProperties through ManagedLedger#asyncOpenCursor. + // ManagedLedger#newNonDurableCursor has no equivalent parameter, so we seed the cursor + // with a post-construction setCursorProperties call. + initPropertiesFuture = cursor.setCursorProperties(subscriptionProperties); + } } else { // if subscription exists, check if it's a durable subscription if (subscription.getCursor() != null && subscription.getCursor().isDurable()) { @@ -1229,11 +1237,19 @@ private CompletableFuture getNonDurableSubscription(Stri } } + final PersistentSubscription finalSubscription = subscription; if (startMessageRollbackDurationSec > 0) { - resetSubscriptionCursor(subscription, subscriptionFuture, startMessageRollbackDurationSec); + initPropertiesFuture.whenComplete((__, ex) -> { + if (ex != null) { + subscriptionFuture.completeExceptionally(ex); + } else { + resetSubscriptionCursor(finalSubscription, subscriptionFuture, + startMessageRollbackDurationSec); + } + }); return subscriptionFuture; } else { - return CompletableFuture.completedFuture(subscription); + return initPropertiesFuture.thenApply(__ -> finalSubscription); } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java index 3b911d0a87c24..c73ead2f6eaf9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.broker.service.persistent; +import static org.assertj.core.api.Assertions.assertThat; +import static org.apache.bookkeeper.mledger.ManagedCursor.CURSOR_INTERNAL_PROPERTY_PREFIX; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; @@ -35,6 +37,7 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import org.apache.bookkeeper.mledger.AsyncCallbacks; @@ -49,6 +52,7 @@ import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.resources.NamespaceResources; import org.apache.pulsar.broker.service.Consumer; +import org.apache.pulsar.broker.service.GetStatsOptions; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.broker.transaction.buffer.impl.InMemTransactionBufferProvider; import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; @@ -60,6 +64,7 @@ import org.apache.pulsar.common.api.proto.CommandSubscribe; import org.apache.pulsar.common.api.proto.TxnAction; import org.apache.pulsar.common.policies.data.Policies; +import org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl; import org.apache.pulsar.transaction.common.exception.TransactionConflictException; import org.awaitility.Awaitility; import org.testng.annotations.AfterMethod; @@ -230,6 +235,31 @@ public void testAcknowledgeUpdateCursorLastActive() throws Exception { assertTrue(persistentSubscription.cursor.getLastActive() > beforeAcknowledgeTimestamp); } + @Test + public void testGetSubscriptionPropertiesReflectsLiveCursorUpdates() throws Exception { + Map backing = new ConcurrentHashMap<>(); + doReturn(backing).when(cursorMock).getCursorProperties(); + doAnswer(inv -> { + backing.put(inv.getArgument(0), inv.getArgument(1)); + return CompletableFuture.completedFuture(null); + }).when(cursorMock).putCursorProperty(any(), any()); + doReturn(false).when(cursorMock).isDurable(); + + assertThat(persistentSubscription.getSubscriptionProperties()).isEmpty(); + + String bucketKey = CURSOR_INTERNAL_PROPERTY_PREFIX + "delayed.bucket_100_100"; + persistentSubscription.getCursor().putCursorProperty(bucketKey, "42").get(); + + Map live = persistentSubscription.getSubscriptionProperties(); + assertThat(live).containsEntry(bucketKey, "42"); + + SubscriptionStatsImpl stats = persistentSubscription + .getStatsAsync(new GetStatsOptions(false, false, false, false, false)).get(); + assertThat(stats.subscriptionProperties) + .isSameAs(live) + .containsEntry(bucketKey, "42"); + } + public static class CustomTransactionPendingAckStoreProvider implements TransactionPendingAckStoreProvider { @Override public CompletableFuture newPendingAckStore(PersistentSubscription subscription) { From 437bb56cbdb2091261c20161ce914c9bff1a9467 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Fri, 10 Jul 2026 10:41:18 +0800 Subject: [PATCH 113/213] [fix][broker] Fix bucket delayed message index metrics reset on scrape (#26171) (cherry picked from commit c240dd2ea2b8365dff921190c5af8ba7380914f6) --- .../bookkeeper/mledger/util/StatsBuckets.java | 28 ++++++++ .../BucketDelayedMessageIndexStats.java | 8 +-- .../BucketDelayedMessageIndexStatsTest.java | 68 +++++++++++++++++++ 3 files changed, 100 insertions(+), 4 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexStatsTest.java diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/StatsBuckets.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/StatsBuckets.java index 60c0a7f6c9d22..ec3a7e00fd390 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/StatsBuckets.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/StatsBuckets.java @@ -60,6 +60,16 @@ public void addValue(long value) { sumCounter.add(value); } + /** + * Snapshots the current values into {@link #getBuckets()}, {@link #getCount()}, and + * {@link #getSum()}, then resets the internal counters to zero. + * + *

Use this for periodic aggregation where the consumer reads values on a fixed + * schedule (e.g. {@code ManagedLedgerMBeanImpl} refreshes every + * {@code managedLedgerStatsPeriodSeconds}). Do not use this for metrics that + * are scraped on demand (e.g. Prometheus {@code /metrics}), because data recorded + * between scrapes is lost after each call — use {@link #snapshot()} instead. + */ public void refresh() { long count = 0; sum = sumCounter.sumThenReset(); @@ -73,6 +83,24 @@ public void refresh() { this.count = count; } + /** + * Snapshots the current cumulative values without resetting the internal counters. + * Use this instead of {@link #refresh()} when the data should survive across reads + * (e.g. Prometheus metrics that are scraped multiple times). + */ + public void snapshot() { + long count = 0; + sum = sumCounter.sum(); + + for (int i = 0; i < buckets.length; i++) { + long value = buckets[i].sum(); + count += value; + values[i] = value; + } + + this.count = count; + } + public void reset() { sum = 0; sumCounter.reset(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexStats.java index 68788c359d560..b9e6c7dc64c7c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexStats.java @@ -47,8 +47,8 @@ enum Type { private static final String BUCKET_TOTAL_NAME = "pulsar_delayed_message_index_bucket_total"; private static final String INDEX_LOADED_NAME = "pulsar_delayed_message_index_loaded"; private static final String SNAPSHOT_SIZE_BYTES_NAME = "pulsar_delayed_message_index_bucket_snapshot_size_bytes"; - private static final String OP_COUNT_NAME = "pulsar_delayed_message_index_bucket_op_count"; - private static final String OP_LATENCY_NAME = "pulsar_delayed_message_index_bucket_op_latency_ms"; + static final String OP_COUNT_NAME = "pulsar_delayed_message_index_bucket_op_count"; + static final String OP_LATENCY_NAME = "pulsar_delayed_message_index_bucket_op_latency_ms"; private final AtomicInteger delayedMessageIndexBucketTotal = new AtomicInteger(); private final AtomicLong delayedMessageIndexLoaded = new AtomicLong(); @@ -75,11 +75,11 @@ public Map genTopicMetricMap() { String[] labels = splitKey(k); String[] labelsAndValues = new String[] {"state", labels[0], "type", labels[1]}; String key = OP_COUNT_NAME + joinKey(labelsAndValues); - metrics.put(key, new TopicMetricBean(OP_COUNT_NAME, count.sumThenReset(), labelsAndValues)); + metrics.put(key, new TopicMetricBean(OP_COUNT_NAME, count.sum(), labelsAndValues)); }); delayedMessageIndexBucketOpLatencyMs.forEach((typeName, statsBuckets) -> { - statsBuckets.refresh(); + statsBuckets.snapshot(); long[] buckets = statsBuckets.getBuckets(); for (int i = 0; i < buckets.length; i++) { long count = buckets[i]; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexStatsTest.java new file mode 100644 index 0000000000000..b1b7d8248cc37 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexStatsTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.delayed.bucket; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import java.util.Map; +import org.apache.pulsar.common.policies.data.stats.TopicMetricBean; +import org.testng.annotations.Test; + +public class BucketDelayedMessageIndexStatsTest { + + private static final String OP_COUNT = BucketDelayedMessageIndexStats.OP_COUNT_NAME; + private static final String OP_LATENCY_COUNT = BucketDelayedMessageIndexStats.OP_LATENCY_NAME + "_count"; + private static final String OP_LATENCY_SUM = BucketDelayedMessageIndexStats.OP_LATENCY_NAME + "_sum"; + + @Test + public void testMetricsAreCumulativeAcrossScrapes() { + BucketDelayedMessageIndexStats stats = new BucketDelayedMessageIndexStats(); + var type = BucketDelayedMessageIndexStats.Type.create; + + stats.recordTriggerEvent(type); + stats.recordSuccessEvent(type, 200); + stats.recordSuccessEvent(type, 300); + + // First scrape + var m1 = stats.genTopicMetricMap(); + assertEquals(get(m1, OP_COUNT, "state", "succeed", "type", "create"), 2); + assertEquals(get(m1, OP_LATENCY_COUNT, "type", "create"), 2); + assertEquals(get(m1, OP_LATENCY_SUM, "type", "create"), 500); + + // Second scrape — must not reset + var m2 = stats.genTopicMetricMap(); + assertEquals(get(m2, OP_COUNT, "state", "succeed", "type", "create"), 2); + assertEquals(get(m2, OP_LATENCY_COUNT, "type", "create"), 2); + assertEquals(get(m2, OP_LATENCY_SUM, "type", "create"), 500); + + // Third event — cumulative + stats.recordSuccessEvent(type, 100); + var m3 = stats.genTopicMetricMap(); + assertEquals(get(m3, OP_COUNT, "state", "succeed", "type", "create"), 3); + assertEquals(get(m3, OP_LATENCY_COUNT, "type", "create"), 3); + assertEquals(get(m3, OP_LATENCY_SUM, "type", "create"), 600); + } + + private static long get(Map metrics, String name, String... labelsAndValues) { + String key = name + BucketDelayedMessageIndexStats.joinKey(labelsAndValues); + TopicMetricBean bean = metrics.get(key); + assertNotNull(bean, "Metric not found: " + key); + return (long) bean.value; + } +} From 49b7d63c47de2ba8dbc87b0ab360b2666039d7c5 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Mon, 8 Jun 2026 18:23:33 +0800 Subject: [PATCH 114/213] [improve][meta] Upgrade Oxia client to 0.8.0 (#25964) --- distribution/server/src/assemble/LICENSE.bin.txt | 4 ++-- pom.xml | 5 +++-- .../java/org/apache/pulsar/metadata/MetadataStoreTest.java | 4 +--- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 3b10881ab2f84..b216c286244b3 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -497,8 +497,8 @@ The Apache Software License, Version 2.0 * Prometheus - io.prometheus-simpleclient_httpserver-0.16.0.jar * Oxia - - io.github.oxia-db-oxia-client-api-0.7.2.jar - - io.github.oxia-db-oxia-client-0.7.2.jar + - io.github.oxia-db-oxia-client-api-0.8.0.jar + - io.github.oxia-db-oxia-client-0.8.0.jar * OpenHFT - net.openhft-zero-allocation-hashing-0.16.jar * Java JSON WebTokens diff --git a/pom.xml b/pom.xml index e3812d3f992a1..697437ea5ec51 100644 --- a/pom.xml +++ b/pom.xml @@ -297,7 +297,8 @@ flexible messaging model and an intuitive client API. 4.5.14 4.4.16 0.7.7 - 0.7.2 + 0.8.0 + 0.7.4 2.0 1.10.12 5.5.0 @@ -1313,7 +1314,7 @@ flexible messaging model and an intuitive client API. io.github.oxia-db oxia-testcontainers - ${oxia.version} + ${oxia-testcontainers.version} diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java index 14810b196929f..3f484ff0d0ff8 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java @@ -32,7 +32,6 @@ import static org.testng.Assert.fail; import io.oxia.client.ClientConfig; import io.oxia.client.api.AsyncOxiaClient; -import io.oxia.client.session.SessionFactory; import io.oxia.client.session.SessionManager; import java.nio.charset.StandardCharsets; import java.time.Duration; @@ -612,8 +611,7 @@ public void testOxiaLoadConfigFromFile() throws Exception { OxiaMetadataStore store = (OxiaMetadataStore) MetadataStoreFactory.create(oxia, config); var client = (AsyncOxiaClient) WhiteboxImpl.getInternalState(store, "client"); var sessionManager = (SessionManager) WhiteboxImpl.getInternalState(client, "sessionManager"); - var sessionFactory = (SessionFactory) WhiteboxImpl.getInternalState(sessionManager, "factory"); - var clientConfig = (ClientConfig) WhiteboxImpl.getInternalState(sessionFactory, "config"); + var clientConfig = (ClientConfig) WhiteboxImpl.getInternalState(sessionManager, "clientConfig"); var sessionTimeout = clientConfig.sessionTimeout(); assertEquals(sessionTimeout, Duration.ofSeconds(60)); } From cd18660db60a58d8ff3c4f2dc0e6aad3bf9532ed Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Wed, 15 Jul 2026 00:55:14 -0700 Subject: [PATCH 115/213] [improve][meta] Upgrade Oxia client to 0.9.4 (#26193) (cherry picked from commit 0ae4d94689d1f01988803c34261789f098da81a6) --- distribution/server/src/assemble/LICENSE.bin.txt | 4 ++-- pom.xml | 2 +- .../apache/pulsar/metadata/impl/oxia/OxiaMetadataStore.java | 5 ----- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index b216c286244b3..37c1476e915ea 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -497,8 +497,8 @@ The Apache Software License, Version 2.0 * Prometheus - io.prometheus-simpleclient_httpserver-0.16.0.jar * Oxia - - io.github.oxia-db-oxia-client-api-0.8.0.jar - - io.github.oxia-db-oxia-client-0.8.0.jar + - io.github.oxia-db-oxia-client-api-0.9.4.jar + - io.github.oxia-db-oxia-client-0.9.4.jar * OpenHFT - net.openhft-zero-allocation-hashing-0.16.jar * Java JSON WebTokens diff --git a/pom.xml b/pom.xml index 697437ea5ec51..5842824ac4be6 100644 --- a/pom.xml +++ b/pom.xml @@ -297,7 +297,7 @@ flexible messaging model and an intuitive client API. 4.5.14 4.4.16 0.7.7 - 0.8.0 + 0.9.4 0.7.4 2.0 1.10.12 diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/oxia/OxiaMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/oxia/OxiaMetadataStore.java index c1e65d4eac1e1..914253e8630f4 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/oxia/OxiaMetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/oxia/OxiaMetadataStore.java @@ -77,10 +77,6 @@ public OxiaMetadataStore( super("oxia-metadata", Objects.requireNonNull(metadataStoreConfig).getOpenTelemetry(), metadataStoreConfig.getNodeSizeStats(), metadataStoreConfig.getNumSerDesThreads()); - var linger = metadataStoreConfig.getBatchingMaxDelayMillis(); - if (!metadataStoreConfig.isBatchingEnabled()) { - linger = 0; - } synchronizer = Optional.ofNullable(metadataStoreConfig.getSynchronizer()); identity = UUID.randomUUID().toString(); OxiaClientBuilder oxiaClientBuilder = OxiaClientBuilder @@ -88,7 +84,6 @@ public OxiaMetadataStore( .clientIdentifier(identity) .namespace(namespace) .sessionTimeout(Duration.ofMillis(metadataStoreConfig.getSessionTimeoutMillis())) - .batchLinger(Duration.ofMillis(linger)) .maxRequestsPerBatch(metadataStoreConfig.getBatchingMaxOperations()); if (StringUtils.isNotBlank(metadataStoreConfig.getConfigFilePath())) { oxiaClientBuilder.loadConfig(metadataStoreConfig.getConfigFilePath()); From 821f09cae0fc210d3c8d33992f3966b5d8282e69 Mon Sep 17 00:00:00 2001 From: zhou zhuohan <843520313@qq.com> Date: Wed, 15 Jul 2026 20:47:57 +0800 Subject: [PATCH 116/213] [fix][client] Fix lookup permit double-release, waiting queue starvation and timeout-response races in ClientCnx (#26143) (cherry picked from commit 8576283da4ad3f3742168fc4bf4a527f31923297) --- .../apache/pulsar/client/impl/ClientCnx.java | 162 ++++++---- .../pulsar/client/impl/ClientCnxTest.java | 298 ++++++++++++++++++ 2 files changed, 393 insertions(+), 67 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index 510ef500adc7d..0a81fdfaeebf9 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -40,7 +40,9 @@ import java.util.List; import java.util.Optional; import java.util.Queue; +import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.Semaphore; @@ -131,6 +133,11 @@ public class ClientCnx extends PulsarHandler { .expectedItems(16) .concurrencyLevel(1) .build(); + // pendingRequests stores all pending request futures but does not preserve the command type, + // so there is no way to distinguish lookup requests from other requests (e.g. producer/consumer creation, + // getTopics, getLastMessageId). This set tracks which requestIds belong to lookup requests, + // so that removePendingRequest can release the lookup semaphore correctly. + private final Set pendingLookupRequestIds = ConcurrentHashMap.newKeySet(); // LookupRequests that waiting in client side. private final Queue>>> waitingLookupRequests; @@ -340,13 +347,16 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { ConnectException e = new ConnectException( "Disconnected from server at " + ctx.channel().remoteAddress()); + // Fail out all waiting lookup requests first, and clear the queue so that + // releasePermitAndDriveWaitingQueue won't dispatch requests on a dead connection. + waitingLookupRequests.forEach(pair -> pair.getRight().getRight().completeExceptionally(e)); + waitingLookupRequests.clear(); // Fail out all the pending ops pendingRequests.forEach((key, future) -> { - if (pendingRequests.remove(key, future) && !future.isDone()) { + if (removePendingRequest(key, future) && !future.isDone()) { future.completeExceptionally(e); } }); - waitingLookupRequests.forEach(pair -> pair.getRight().getRight().completeExceptionally(e)); // Notify all attached producers/consumers so they have a chance to reconnect producers.forEach((id, producer) -> producer.connectionClosed(this, Optional.empty(), Optional.empty())); @@ -354,8 +364,6 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { transactionMetaStoreHandlers.forEach((id, handler) -> handler.connectionClosed(this)); topicListWatchers.forEach((__, watcher) -> watcher.connectionClosed(this)); - waitingLookupRequests.clear(); - producers.clear(); consumers.clear(); topicListWatchers.clear(); @@ -516,8 +524,9 @@ protected void handleSendReceipt(CommandSendReceipt sendReceipt) { protected void handleAckResponse(CommandAckResponse ackResponse) { checkArgument(state == State.Ready); checkArgument(ackResponse.getRequestId() >= 0); - CompletableFuture completableFuture = pendingRequests.remove(ackResponse.getRequestId()); - if (completableFuture != null && !completableFuture.isDone()) { + long requestId = ackResponse.getRequestId(); + CompletableFuture completableFuture = pendingRequests.get(requestId); + if (completableFuture != null && removePendingRequest(requestId, completableFuture)) { if (!ackResponse.hasError()) { completableFuture.complete(null); } else { @@ -567,8 +576,8 @@ protected void handleSuccess(CommandSuccess success) { log.debug("{} Received success response from server: {}", ctx.channel(), success.getRequestId()); } long requestId = success.getRequestId(); - CompletableFuture requestFuture = pendingRequests.remove(requestId); - if (requestFuture != null) { + CompletableFuture requestFuture = pendingRequests.get(requestId); + if (requestFuture != null && removePendingRequest(requestId, requestFuture)) { requestFuture.complete(null); } else { duplicatedResponseCounter.incrementAndGet(); @@ -586,8 +595,8 @@ protected void handleGetLastMessageIdSuccess(CommandGetLastMessageIdResponse suc } long requestId = success.getRequestId(); CompletableFuture requestFuture = - (CompletableFuture) pendingRequests.remove(requestId); - if (requestFuture != null) { + (CompletableFuture) pendingRequests.get(requestId); + if (requestFuture != null && removePendingRequest(requestId, requestFuture)) { requestFuture.complete(new CommandGetLastMessageIdResponse().copyFrom(success)); } else { duplicatedResponseCounter.incrementAndGet(); @@ -618,8 +627,8 @@ protected void handleProducerSuccess(CommandProducerSuccess success) { } CompletableFuture requestFuture = - (CompletableFuture) pendingRequests.remove(requestId); - if (requestFuture != null) { + (CompletableFuture) pendingRequests.get(requestId); + if (requestFuture != null && removePendingRequest(requestId, requestFuture)) { ProducerResponse pr = new ProducerResponse(success.getProducerName(), success.getLastSequenceId(), success.getSchemaVersion(), @@ -640,9 +649,10 @@ protected void handleLookupResponse(CommandLookupTopicResponse lookupResult) { } long requestId = lookupResult.getRequestId(); - CompletableFuture requestFuture = getAndRemovePendingLookupRequest(requestId); + CompletableFuture requestFuture = + (CompletableFuture) pendingRequests.get(requestId); - if (requestFuture != null) { + if (requestFuture != null && removePendingRequest(requestId, requestFuture)) { if (requestFuture.isCompletedExceptionally()) { if (log.isDebugEnabled()) { log.debug("{} Request {} already timed-out", ctx.channel(), lookupResult.getRequestId()); @@ -667,6 +677,7 @@ protected void handleLookupResponse(CommandLookupTopicResponse lookupResult) { requestFuture.complete(new LookupDataResult(lookupResult)); } } else { + duplicatedResponseCounter.incrementAndGet(); log.warn("{} Received unknown request id from server: {}", ctx.channel(), lookupResult.getRequestId()); } } @@ -682,9 +693,10 @@ protected void handlePartitionResponse(CommandPartitionedTopicMetadataResponse l } long requestId = lookupResult.getRequestId(); - CompletableFuture requestFuture = getAndRemovePendingLookupRequest(requestId); + CompletableFuture requestFuture = + (CompletableFuture) pendingRequests.get(requestId); - if (requestFuture != null) { + if (requestFuture != null && removePendingRequest(requestId, requestFuture)) { if (requestFuture.isCompletedExceptionally()) { if (log.isDebugEnabled()) { log.debug("{} Request {} already timed-out", ctx.channel(), lookupResult.getRequestId()); @@ -709,6 +721,7 @@ protected void handlePartitionResponse(CommandPartitionedTopicMetadataResponse l requestFuture.complete(new LookupDataResult(lookupResult.getPartitions())); } } else { + duplicatedResponseCounter.incrementAndGet(); log.warn("{} Received unknown request id from server: {}", ctx.channel(), lookupResult.getRequestId()); } } @@ -750,39 +763,61 @@ protected void handleTopicMigrated(CommandTopicMigrated commandTopicMigrated) { // caller of this method needs to be protected under pendingLookupRequestSemaphore private void addPendingLookupRequests(long requestId, TimedCompletableFuture future) { + pendingLookupRequestIds.add(requestId); pendingRequests.put(requestId, future); requestTimeoutQueue.add(new RequestTime(requestId, RequestType.Lookup)); } - private CompletableFuture getAndRemovePendingLookupRequest(long requestId) { - CompletableFuture result = - (CompletableFuture) pendingRequests.remove(requestId); - if (result != null) { - Pair>> firstOneWaiting = - waitingLookupRequests.poll(); - if (firstOneWaiting != null) { - maxLookupRequestSemaphore.release(); - // schedule a new lookup in. - eventLoopGroup.execute(() -> { - long newId = firstOneWaiting.getLeft(); - TimedCompletableFuture newFuture = firstOneWaiting.getRight().getRight(); - addPendingLookupRequests(newId, newFuture); - ctx.writeAndFlush(firstOneWaiting.getRight().getLeft()).addListener(writeFuture -> { - if (!writeFuture.isSuccess()) { - log.warn("{} Failed to send request {} to broker: {}", ctx.channel(), newId, + /** + * Release the lookup semaphore permit and drive the waiting queue. + * This is the single centralized primitive for permit release/transfer. + */ + private void releasePermitAndDriveWaitingQueue() { + Pair>> firstOneWaiting = + waitingLookupRequests.poll(); + if (firstOneWaiting != null) { + maxLookupRequestSemaphore.release(); + // schedule a new lookup in. + eventLoopGroup.execute(() -> { + long newId = firstOneWaiting.getLeft(); + TimedCompletableFuture newFuture = firstOneWaiting.getRight().getRight(); + addPendingLookupRequests(newId, newFuture); + ctx.writeAndFlush(firstOneWaiting.getRight().getLeft()).addListener(writeFuture -> { + if (!writeFuture.isSuccess()) { + log.warn("{} Failed to send request {} to broker: {}", ctx.channel(), newId, writeFuture.cause().getMessage()); - getAndRemovePendingLookupRequest(newId); + if (removePendingRequest(newId, newFuture)) { newFuture.completeExceptionally(writeFuture.cause()); } - }); + } }); - } else { - pendingLookupRequestSemaphore.release(); - } + }); } else { - duplicatedResponseCounter.incrementAndGet(); + pendingLookupRequestSemaphore.release(); } - return result; + } + + /** + * Unified cleanup primitive for all pending requests. + * Uses pendingRequests.remove(requestId, expectedFuture) as the single CAS contention point. + * Only the thread that successfully removes the entry from pendingRequests owns the cleanup + * responsibility (including permit release for lookup requests). + * + * @param requestId the request ID to remove + * @param expectedFuture the expected future value for CAS comparison + * @return true if this call successfully obtained ownership and performed cleanup + */ + private boolean removePendingRequest(long requestId, CompletableFuture expectedFuture) { + // CAS: only one thread can successfully remove from pendingRequests + if (!pendingRequests.remove(requestId, expectedFuture)) { + return false; + } + // Won the CAS. If it's a lookup request, release the permit and drive waiting queue. + if (pendingLookupRequestIds.contains(requestId)) { + releasePermitAndDriveWaitingQueue(); + pendingLookupRequestIds.remove(requestId); + } + return true; } @Override @@ -834,8 +869,8 @@ protected void handleError(CommandError error) { log.error("Get not allowed error, {}", error.getMessage()); connectionFuture.completeExceptionally(new PulsarClientException.NotAllowedException(error.getMessage())); } - CompletableFuture requestFuture = pendingRequests.remove(requestId); - if (requestFuture != null) { + CompletableFuture requestFuture = pendingRequests.get(requestId); + if (requestFuture != null && removePendingRequest(requestId, requestFuture)) { requestFuture.completeExceptionally( getPulsarClientException(error.getError(), buildError(error.getRequestId(), error.getMessage()))); @@ -925,20 +960,14 @@ public CompletableFuture newLookup(ByteBuf request, long reque TimedCompletableFuture future = new TimedCompletableFuture<>(); if (pendingLookupRequestSemaphore.tryAcquire()) { - future.whenComplete((lookupDataResult, throwable) -> { - if (throwable instanceof ConnectException - || throwable instanceof PulsarClientException.LookupException - || FutureUtil.unwrapCompletionException(throwable) instanceof TimeoutException) { - pendingLookupRequestSemaphore.release(); - } - }); addPendingLookupRequests(requestId, future); ctx.writeAndFlush(request).addListener(writeFuture -> { if (!writeFuture.isSuccess()) { log.warn("{} Failed to send request {} to broker: {}", ctx.channel(), requestId, writeFuture.cause().getMessage()); - getAndRemovePendingLookupRequest(requestId); - future.completeExceptionally(writeFuture.cause()); + if (removePendingRequest(requestId, future)) { + future.completeExceptionally(writeFuture.cause()); + } } }); } else { @@ -990,8 +1019,8 @@ protected void handleGetTopicsOfNamespaceSuccess(CommandGetTopicsOfNamespaceResp } CompletableFuture requestFuture = - (CompletableFuture) pendingRequests.remove(requestId); - if (requestFuture != null) { + (CompletableFuture) pendingRequests.get(requestId); + if (requestFuture != null && removePendingRequest(requestId, requestFuture)) { requestFuture.complete(new GetTopicsResult(topics, success.hasTopicsHash() ? success.getTopicsHash() : null, success.isFiltered(), @@ -1009,8 +1038,8 @@ protected void handleGetSchemaResponse(CommandGetSchemaResponse commandGetSchema long requestId = commandGetSchemaResponse.getRequestId(); CompletableFuture future = - (CompletableFuture) pendingRequests.remove(requestId); - if (future == null) { + (CompletableFuture) pendingRequests.get(requestId); + if (future == null || !removePendingRequest(requestId, future)) { duplicatedResponseCounter.incrementAndGet(); log.warn("{} Received unknown request id from server: {}", ctx.channel(), requestId); return; @@ -1023,10 +1052,11 @@ protected void handleGetOrCreateSchemaResponse(CommandGetOrCreateSchemaResponse checkArgument(state == State.Ready); long requestId = commandGetOrCreateSchemaResponse.getRequestId(); CompletableFuture future = - (CompletableFuture) pendingRequests.remove(requestId); - if (future == null) { + (CompletableFuture) pendingRequests.get(requestId); + if (future == null || !removePendingRequest(requestId, future)) { duplicatedResponseCounter.incrementAndGet(); - log.warn("{} Received unknown request id from server: {}", ctx.channel(), requestId); + log.warn("{} Received unknown request id from server: {}", ctx.channel(), + commandGetOrCreateSchemaResponse.getRequestId()); return; } future.complete(new CommandGetOrCreateSchemaResponse().copyFrom(commandGetOrCreateSchemaResponse)); @@ -1059,7 +1089,7 @@ private void sendRequestAndHandleTimeout(ByteBuf requestMessage, long reques pendingRequests.put(requestId, future); (flush ? ctx.writeAndFlush(requestMessage) : ctx.write(requestMessage)).addListener(writeFuture -> { if (!writeFuture.isSuccess()) { - if (pendingRequests.remove(requestId, future) && !future.isDone()) { + if (removePendingRequest(requestId, future)) { log.warn("{} Failed to send {} to broker: {}", ctx.channel(), requestType.getDescription(), writeFuture.cause().getMessage()); future.completeExceptionally(writeFuture.cause()); @@ -1179,9 +1209,8 @@ protected void handleTcClientConnectResponse(CommandTcClientConnectResponse resp + "from server: {}", ctx.channel(), response.getRequestId()); } long requestId = response.getRequestId(); - CompletableFuture requestFuture = pendingRequests.remove(requestId); - - if (requestFuture != null && !requestFuture.isDone()) { + CompletableFuture requestFuture = pendingRequests.get(requestId); + if (requestFuture != null && removePendingRequest(requestId, requestFuture)) { if (!response.hasError()) { requestFuture.complete(null); } else { @@ -1251,8 +1280,8 @@ protected void handleCommandWatchTopicListSuccess(CommandWatchTopicListSuccess c } long requestId = commandWatchTopicListSuccess.getRequestId(); CompletableFuture requestFuture = - (CompletableFuture) pendingRequests.remove(requestId); - if (requestFuture != null) { + (CompletableFuture) pendingRequests.get(requestId); + if (requestFuture != null && removePendingRequest(requestId, requestFuture)) { requestFuture.complete(new CommandWatchTopicListSuccess().copyFrom(commandWatchTopicListSuccess)); } else { duplicatedResponseCounter.incrementAndGet(); @@ -1472,10 +1501,9 @@ private void checkRequestTimeout() { continue; } TimedCompletableFuture requestFuture = pendingRequests.get(request.requestId); - if (requestFuture != null - && !requestFuture.hasGotResponse()) { - pendingRequests.remove(request.requestId, requestFuture); - if (!requestFuture.isDone()) { + if (requestFuture != null && !requestFuture.hasGotResponse()) { + boolean removed = removePendingRequest(request.requestId, requestFuture); + if (removed) { String timeoutMessage = String.format( "%s timeout {'durationMs': '%d', 'reqId':'%d', 'remote':'%s', 'local':'%s'}", request.requestType.getDescription(), operationTimeoutMs, diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java index c0a75b09ccea1..20de6eacf55bf 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java @@ -37,6 +37,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.PulsarClientException.BrokerMetadataException; @@ -46,6 +47,7 @@ import org.apache.pulsar.common.api.proto.CommandCloseProducer; import org.apache.pulsar.common.api.proto.CommandConnected; import org.apache.pulsar.common.api.proto.CommandError; +import org.apache.pulsar.common.api.proto.CommandLookupTopicResponse; import org.apache.pulsar.common.api.proto.CommandWatchTopicListSuccess; import org.apache.pulsar.common.api.proto.CommandWatchTopicUpdate; import org.apache.pulsar.common.api.proto.ServerError; @@ -371,6 +373,302 @@ public void testUpdateWatcher() { }); } + /** + * Test that when a lookup request times out, the semaphore is properly released + * so that subsequent lookup requests can still be sent. + */ + @Test + public void testLookupTimeoutReleasesSemaphore() throws Exception { + EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, + new DefaultThreadFactory("testLookupTimeoutReleasesSemaphore")); + try { + ClientConfigurationData conf = new ClientConfigurationData(); + conf.setOperationTimeoutMs(10); + conf.setKeepAliveIntervalSeconds(0); + ClientCnx cnx = new ClientCnx(InstrumentProvider.NOOP, conf, eventLoop); + ChannelHandlerContext ctx = ClientTestFixtures.mockChannelHandlerContext(); + cnx.channelActive(ctx); + + int initialPermits = cnx.getPendingLookupRequestSemaphore().availablePermits(); + + // Send a lookup request that will time out + CompletableFuture future = + cnx.newLookup(null, 1L); + + // Wait for the timeout to trigger + try { + future.get(2, TimeUnit.SECONDS); + fail("Should have timed out"); + } catch (Exception e) { + assertTrue(e.getCause() instanceof PulsarClientException.TimeoutException); + } + + // Verify semaphore is released back to initial permits + Awaitility.await().atMost(2, TimeUnit.SECONDS).untilAsserted(() -> { + assertEquals(cnx.getPendingLookupRequestSemaphore().availablePermits(), initialPermits); + }); + } finally { + eventLoop.shutdownGracefully().sync(); + } + } + + /** + * Test that when a lookup request times out, waiting lookup requests in the queue + * are properly dispatched (the waiting queue is driven). + */ + @Test + public void testLookupTimeoutDrivesWaitingQueue() throws Exception { + EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, + new DefaultThreadFactory("testLookupTimeoutDrivesWaitingQueue")); + try { + ClientConfigurationData conf = new ClientConfigurationData(); + // concurrentLookupRequest=50 by default, maxLookupRequest=50000 by default + conf.setOperationTimeoutMs(50); + conf.setKeepAliveIntervalSeconds(0); + conf.setConcurrentLookupRequest(1); // Only allow 1 concurrent lookup + conf.setMaxLookupRequest(10); // Allow up to 10 total (9 waiting) + ClientCnx cnx = new ClientCnx(InstrumentProvider.NOOP, conf, eventLoop); + ChannelHandlerContext ctx = ClientTestFixtures.mockChannelHandlerContext(); + cnx.channelActive(ctx); + cnx.state = ClientCnx.State.Ready; + + // First lookup occupies the only concurrent slot + CompletableFuture firstFuture = + cnx.newLookup(null, 1L); + + // Second lookup should go into the waiting queue + CompletableFuture secondFuture = + cnx.newLookup(null, 2L); + + // The second future should not be completed yet (it's waiting) + assertFalse(secondFuture.isDone()); + + // Wait for the first request to time out - this should drive the waiting queue + // and dispatch the second request + try { + firstFuture.get(2, TimeUnit.SECONDS); + fail("First future should have timed out"); + } catch (Exception e) { + assertTrue(e.getCause() instanceof PulsarClientException.TimeoutException); + } + + // After the first request times out, the second request should have been + // dispatched from the waiting queue (it will also eventually time out) + try { + secondFuture.get(2, TimeUnit.SECONDS); + fail("Second future should have timed out"); + } catch (Exception e) { + assertTrue(e.getCause() instanceof PulsarClientException.TimeoutException); + } + + // Verify all semaphores are properly released + Awaitility.await().atMost(2, TimeUnit.SECONDS).untilAsserted(() -> { + assertEquals(cnx.getPendingLookupRequestSemaphore().availablePermits(), 1); + }); + } finally { + eventLoop.shutdownGracefully().sync(); + } + } + + /** + * Test that when multiple lookup requests time out, all waiting requests in the queue + * are eventually dispatched and no semaphore leak occurs. + */ + @Test + public void testMultipleLookupTimeoutsNoSemaphoreLeak() throws Exception { + EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, + new DefaultThreadFactory("testMultipleLookupTimeoutsNoSemaphoreLeak")); + try { + ClientConfigurationData conf = new ClientConfigurationData(); + conf.setOperationTimeoutMs(30); + conf.setKeepAliveIntervalSeconds(0); + conf.setConcurrentLookupRequest(2); // Allow 2 concurrent lookups + conf.setMaxLookupRequest(10); // Allow up to 10 total (8 waiting) + ClientCnx cnx = new ClientCnx(InstrumentProvider.NOOP, conf, eventLoop); + ChannelHandlerContext ctx = ClientTestFixtures.mockChannelHandlerContext(); + cnx.channelActive(ctx); + cnx.state = ClientCnx.State.Ready; + + // Send 5 lookup requests: 2 will be concurrent, 3 will be in waiting queue + CompletableFuture[] futures = new CompletableFuture[5]; + for (int i = 0; i < 5; i++) { + futures[i] = cnx.newLookup(null, i + 1L); + } + + // Wait for all futures to complete (all should time out eventually) + for (int i = 0; i < 5; i++) { + try { + futures[i].get(5, TimeUnit.SECONDS); + fail("Future " + i + " should have timed out"); + } catch (Exception e) { + assertTrue(e.getCause() instanceof PulsarClientException.TimeoutException, + "Future " + i + " should fail with TimeoutException but got: " + e.getCause()); + } + } + + // Verify all semaphore permits are released (no leak) + Awaitility.await().atMost(2, TimeUnit.SECONDS).untilAsserted(() -> { + assertEquals(cnx.getPendingLookupRequestSemaphore().availablePermits(), 2); + }); + } finally { + eventLoop.shutdownGracefully().sync(); + } + } + + /** + * Test that when a failed lookup response is received, + * the semaphore permit is properly released via the unified cleanup primitive + * (removePendingRequest) without double-release. + */ + @Test + public void testFailedLookupResponseReleasesSemaphore() throws Exception { + EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, + new DefaultThreadFactory("testFailedLookupResponseReleasesSemaphore")); + try { + ClientConfigurationData conf = new ClientConfigurationData(); + conf.setOperationTimeoutMs(10_000); + conf.setKeepAliveIntervalSeconds(0); + ClientCnx cnx = new ClientCnx(InstrumentProvider.NOOP, conf, eventLoop); + ChannelHandlerContext ctx = ClientTestFixtures.mockChannelHandlerContext(); + cnx.channelActive(ctx); + cnx.state = ClientCnx.State.Ready; + + int initialPermits = cnx.getPendingLookupRequestSemaphore().availablePermits(); + + // Send a lookup request + CompletableFuture future = + cnx.newLookup(null, 1L); + + // Simulate a failed lookup response (CommandLookupTopicResponse with Failed status) + CommandLookupTopicResponse lookupResponse = new CommandLookupTopicResponse(); + lookupResponse.setRequestId(1L); + lookupResponse.setResponse(CommandLookupTopicResponse.LookupType.Failed); + lookupResponse.setError(ServerError.ServiceNotReady); + lookupResponse.setMessage("Service not ready"); + cnx.handleLookupResponse(lookupResponse); + + // Verify the future completed exceptionally + try { + future.get(2, TimeUnit.SECONDS); + fail("Should have failed"); + } catch (Exception e) { + // expected + } + + // Verify semaphore is released back to initial permits (no leak, no double-release) + // Previously this would cause availablePermits to be initialPermits+1 due to double-release + Awaitility.await().atMost(2, TimeUnit.SECONDS).untilAsserted(() -> { + assertEquals(cnx.getPendingLookupRequestSemaphore().availablePermits(), initialPermits); + }); + } finally { + eventLoop.shutdownGracefully().sync(); + } + } + + /** + * Test that when the connection is closed while an active lookup request is pending, + * the semaphore permit is properly released. + */ + @Test + public void testActiveRequestConnectionCloseReleasesSemaphore() throws Exception { + EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, + new DefaultThreadFactory("testActiveRequestConnectionCloseReleasesSemaphore")); + try { + ClientConfigurationData conf = new ClientConfigurationData(); + conf.setOperationTimeoutMs(10_000); + conf.setKeepAliveIntervalSeconds(0); + ClientCnx cnx = new ClientCnx(InstrumentProvider.NOOP, conf, eventLoop); + ChannelHandlerContext ctx = ClientTestFixtures.mockChannelHandlerContext(); + cnx.channelActive(ctx); + + int initialPermits = cnx.getPendingLookupRequestSemaphore().availablePermits(); + + // Send a lookup request + CompletableFuture future = + cnx.newLookup(null, 1L); + + // Simulate connection close + cnx.channelInactive(ctx); + + // Verify the future completed exceptionally with ConnectException + try { + future.get(2, TimeUnit.SECONDS); + fail("Should have failed"); + } catch (Exception e) { + assertTrue(e.getCause() instanceof PulsarClientException.ConnectException); + } + + // Verify semaphore is released back to initial permits + Awaitility.await().atMost(2, TimeUnit.SECONDS).untilAsserted(() -> { + assertEquals(cnx.getPendingLookupRequestSemaphore().availablePermits(), initialPermits); + }); + } finally { + eventLoop.shutdownGracefully().sync(); + } + } + + /** + * Test that when the connection is closed while a promoted waiting request is active, + * the semaphore permit is properly released and no leak occurs. + */ + @Test + public void testPromotedWaitingRequestConnectionCloseReleasesSemaphore() throws Exception { + EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, + new DefaultThreadFactory("testPromotedWaitingRequestConnectionCloseReleasesSemaphore")); + try { + ClientConfigurationData conf = new ClientConfigurationData(); + conf.setOperationTimeoutMs(10_000); + conf.setKeepAliveIntervalSeconds(0); + conf.setConcurrentLookupRequest(1); // Only allow 1 concurrent lookup + conf.setMaxLookupRequest(10); + ClientCnx cnx = new ClientCnx(InstrumentProvider.NOOP, conf, eventLoop); + ChannelHandlerContext ctx = ClientTestFixtures.mockChannelHandlerContext(); + cnx.channelActive(ctx); + cnx.state = ClientCnx.State.Ready; + + // First lookup occupies the only concurrent slot + CompletableFuture firstFuture = + cnx.newLookup(null, 1L); + + // Second lookup goes into the waiting queue + CompletableFuture secondFuture = + cnx.newLookup(null, 2L); + + // Simulate a failed lookup response for the first request, which promotes the second + CommandLookupTopicResponse failedResponse = new CommandLookupTopicResponse(); + failedResponse.setRequestId(1L); + failedResponse.setResponse(CommandLookupTopicResponse.LookupType.Failed); + failedResponse.setError(ServerError.ServiceNotReady); + failedResponse.setMessage("Service not ready"); + cnx.handleLookupResponse(failedResponse); + + // Verify the first future completed exceptionally + assertTrue(firstFuture.isCompletedExceptionally()); + + // Wait for the second request to be promoted from the waiting queue + Awaitility.await().atMost(2, TimeUnit.SECONDS) + .until(() -> cnx.pendingRequests.containsKey(2L)); + + // Now close the connection while the promoted request is active + cnx.channelInactive(ctx); + + // Verify the second future completed exceptionally + try { + secondFuture.get(2, TimeUnit.SECONDS); + fail("Second future should have failed"); + } catch (Exception e) { + assertTrue(e.getCause() instanceof PulsarClientException.ConnectException); + } + + // Verify semaphore is released (1 permit for concurrentLookupRequest=1) + Awaitility.await().atMost(2, TimeUnit.SECONDS).untilAsserted(() -> { + assertEquals(cnx.getPendingLookupRequestSemaphore().availablePermits(), 1); + }); + } finally { + eventLoop.shutdownGracefully().sync(); + } + } + private void withConnection(String testName, Consumer test) { ThreadFactory threadFactory = new DefaultThreadFactory(testName); EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, threadFactory); From 33aa6395c1afa61c02bb1ad45c62cf1b56ca75ed Mon Sep 17 00:00:00 2001 From: Yan Zhao Date: Thu, 16 Jul 2026 11:53:05 +0800 Subject: [PATCH 117/213] [improve][broker] Skip system cursor when check inactive cursor. (#26149) (cherry picked from commit fe61afda2d5bf4cb2de019cca47ef78cd1e350f1) --- .../pulsar/broker/service/persistent/PersistentTopic.java | 2 +- .../apache/pulsar/broker/service/PersistentTopicTest.java | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index b236762ea12f3..a882d8a6dbcd8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -3594,7 +3594,7 @@ public void checkInactiveSubscriptions(long expirationTimeMillis) { subscriptions.forEach((subName, sub) -> { if (sub.dispatcher != null && sub.dispatcher.isConsumerConnected() || sub.isReplicated() - || isCompactionSubscription(subName)) { + || isSystemCursor(subName)) { return; } if (System.currentTimeMillis() - sub.cursor.getLastActive() > expirationTimeMillis) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java index 70a027cdd625b..f5ec69e43e544 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java @@ -2032,6 +2032,7 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { @Test public void testCheckInactiveSubscriptions() throws Exception { + pulsarTestContext.getConfig().setAdditionalSystemCursorNames(Set.of("additionalSystemCursor")); PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService); final var subscriptions = new ConcurrentHashMap(); @@ -2050,6 +2051,11 @@ public void testCheckInactiveSubscriptions() throws Exception { spyWithClassAndConstructorArgsRecordingInvocations(PersistentSubscription.class, topic, "nonDeletableSubscription2", cursorMock, true); subscriptions.put(nonDeletableSubscription2.getName(), nonDeletableSubscription2); + // This subscription is an additional system cursor. + PersistentSubscription nonDeletableSubscription3 = + spyWithClassAndConstructorArgsRecordingInvocations(PersistentSubscription.class, topic, + "additionalSystemCursor", cursorMock, false); + subscriptions.put(nonDeletableSubscription3.getName(), nonDeletableSubscription3); Field field = topic.getClass().getDeclaredField("subscriptions"); field.setAccessible(true); @@ -2076,6 +2082,7 @@ public void testCheckInactiveSubscriptions() throws Exception { verify(nonDeletableSubscription1, times(0)).delete(); verify(deletableSubscription1, times(1)).delete(); verify(nonDeletableSubscription2, times(0)).delete(); + verify(nonDeletableSubscription3, times(0)).delete(); } @Test From 2971a104c8d8bc8ed6ff982a99f7c84528675e84 Mon Sep 17 00:00:00 2001 From: void-ptr974 Date: Thu, 16 Jul 2026 15:19:13 +0800 Subject: [PATCH 118/213] [fix][broker] Prevent partition expansion from inheriting delayed-delivery bucket state (#26179) (cherry picked from commit 8af7868e7d11490e8463a5435a9311cb5b08ab5f) --- .../admin/impl/PersistentTopicsBase.java | 23 +++++++- .../broker/admin/IncrementPartitionsTest.java | 36 ++++++++++++ .../persistent/BucketDelayedDeliveryTest.java | 57 +++++++++++++++++++ 3 files changed, 115 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index fac2c6e249ff1..14e4b03756a95 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.admin.impl; +import static org.apache.bookkeeper.mledger.ManagedCursor.CURSOR_INTERNAL_PROPERTY_PREFIX; import static org.apache.pulsar.common.api.proto.CompressionType.NONE; import static org.apache.pulsar.common.naming.SystemTopicNames.isSystemTopic; import static org.apache.pulsar.common.naming.SystemTopicNames.isTransactionCoordinatorAssign; @@ -464,6 +465,9 @@ protected CompletableFuture internalCreateNonPartitionedTopicAsync(boolean // We must not re-create non-durable subscriptions on the new partitions .stream().filter(entry -> entry.getValue().isDurable()) .map(entry -> { + Map subscriptionProperties = + filterSubscriptionPropertiesForPartitionExpansion( + entry.getValue().getSubscriptionProperties()); final List> innerFutures = new ArrayList<>(expectPartitions); for (int i = 0; i < expectPartitions; i++) { @@ -471,7 +475,7 @@ protected CompletableFuture internalCreateNonPartitionedTopicAsync(boolean topicName.getPartition(i).toString(), entry.getKey(), MessageId.earliest, entry.getValue().isReplicated(), - entry.getValue().getSubscriptionProperties()) + subscriptionProperties) .exceptionally(ex -> { Throwable rc = FutureUtil.unwrapCompletionException(ex); @@ -533,6 +537,23 @@ protected CompletableFuture internalCreateNonPartitionedTopicAsync(boolean }); } + /** + * Returns the properties that can be applied to a subscription on a newly created partition. + * + *

Cursor internal properties belong to the source partition and must not be copied. Keep all filtering + * rules here so partition expansion has a single property-copy boundary.

+ */ + private static Map filterSubscriptionPropertiesForPartitionExpansion( + Map subscriptionProperties) { + if (subscriptionProperties == null) { + return null; + } + + Map filteredProperties = new HashMap<>(subscriptionProperties); + filteredProperties.keySet().removeIf(key -> key.startsWith(CURSOR_INTERNAL_PROPERTY_PREFIX)); + return filteredProperties; + } + private CompletableFuture> getReplicationClusters() { return namespaceResources().getPoliciesAsync(namespaceName).thenCompose(optionalPolicies -> { if (optionalPolicies.isEmpty()) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/IncrementPartitionsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/IncrementPartitionsTest.java index 091f4ae6c07df..4b2938982eafa 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/IncrementPartitionsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/IncrementPartitionsTest.java @@ -18,7 +18,9 @@ */ package org.apache.pulsar.broker.admin; +import static org.apache.bookkeeper.mledger.ManagedCursor.CURSOR_INTERNAL_PROPERTY_PREFIX; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -30,6 +32,7 @@ import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.admin.AdminApiTest.MockedPulsarService; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; @@ -150,6 +153,39 @@ public void testIncrementPartitionsOfTopicWithSubscriptionProperties() throws Ex Assert.assertEquals(properties, subscriptionProperties); } + @Test + public void testIncrementPartitionsDoesNotCopyInternalCursorProperties() throws Exception { + String partitionedTopicName = UUID.randomUUID() + + "-testIncrementPartitionsDoesNotCopyInternalCursorProperties"; + String subscriptionName = "sub-1"; + Map subscriptionProperties = Map.of("property", "value"); + + admin.topics().createPartitionedTopic(partitionedTopicName, 1); + @Cleanup + Consumer consumer = pulsarClient.newConsumer() + .topic(partitionedTopicName) + .subscriptionName(subscriptionName) + .subscriptionProperties(subscriptionProperties) + .subscribe(); + + String sourcePartition = TopicName.get(partitionedTopicName).getPartition(0).toString(); + PersistentTopic sourceTopic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(sourcePartition) + .orElseThrow(); + String internalProperty = CURSOR_INTERNAL_PROPERTY_PREFIX + "test"; + sourceTopic.getSubscription(subscriptionName).getCursor() + .putCursorProperty(internalProperty, "internal-value").get(); + + admin.topics().updatePartitionedTopic(partitionedTopicName, 2); + + String newPartition = TopicName.get(partitionedTopicName).getPartition(1).toString(); + PersistentTopic newTopic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(newPartition) + .orElseThrow(); + Map newCursorProperties = newTopic.getSubscription(subscriptionName) + .getCursor().getCursorProperties(); + assertEquals(newCursorProperties, subscriptionProperties); + assertFalse(newCursorProperties.containsKey(internalProperty)); + } + @Test public void testIncrementPartitionsWithNoSubscriptions() throws Exception { final String partitionedTopicName = diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java index 3af7b1b4e0cf3..84020f6aea778 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java @@ -22,6 +22,7 @@ import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.Metric; import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.parseMetrics; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import com.google.common.collect.Multimap; @@ -48,6 +49,7 @@ import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.common.naming.TopicName; import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterClass; @@ -126,6 +128,61 @@ public void testBucketDelayedDeliveryWithAllConsumersDisconnecting() throws Exce Assert.assertEquals(bucketKeys, bucketKeys2); } + @Test + public void testIncrementPartitionsDoesNotCopyBucketDelayedDeliveryState() throws Exception { + String topic = BrokerTestUtil.newUniqueName("persistent://public/default/testBucketStatePartitionExpansion"); + String subscriptionName = "sub"; + admin.topics().createPartitionedTopic(topic, 1); + String sourcePartition = TopicName.get(topic).getPartition(0).toString(); + + @Cleanup + Consumer sourceConsumer = pulsarClient.newConsumer(Schema.STRING) + .topic(sourcePartition) + .subscriptionName(subscriptionName) + .subscriptionType(SubscriptionType.Shared) + .subscribe(); + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(sourcePartition) + .enableBatching(false) + .create(); + + for (int i = 0; i < 1000; i++) { + producer.newMessage().value("msg").deliverAfter(1, TimeUnit.HOURS).send(); + } + + Dispatcher sourceDispatcher = pulsar.getBrokerService().getTopicReference(sourcePartition) + .get().getSubscription(subscriptionName).getDispatcher(); + Awaitility.await().untilAsserted( + () -> Assert.assertEquals(sourceDispatcher.getNumberOfDelayedMessages(), 1000)); + List bucketKeys = ((AbstractPersistentDispatcherMultipleConsumers) sourceDispatcher) + .getCursor().getCursorProperties().keySet().stream() + .filter(key -> key.startsWith(CURSOR_INTERNAL_PROPERTY_PREFIX + "delayed.bucket")).toList(); + assertFalse(bucketKeys.isEmpty()); + + admin.topics().updatePartitionedTopic(topic, 2); + + String newPartition = TopicName.get(topic).getPartition(1).toString(); + PersistentTopic newTopic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(newPartition) + .orElseThrow(); + Map newCursorProperties = newTopic.getSubscription(subscriptionName) + .getCursor().getCursorProperties(); + assertTrue(newCursorProperties == null + || newCursorProperties.keySet().stream().noneMatch(bucketKeys::contains)); + + @Cleanup + Consumer newPartitionConsumer = pulsarClient.newConsumer(Schema.STRING) + .topic(newPartition) + .subscriptionName(subscriptionName) + .subscriptionType(SubscriptionType.Shared) + .subscribe(); + Dispatcher newPartitionDispatcher = newTopic.getSubscription(subscriptionName).getDispatcher(); + Awaitility.await().untilAsserted( + () -> assertEquals(newPartitionDispatcher.getNumberOfDelayedMessages(), 0)); + assertTrue(((AbstractPersistentDispatcherMultipleConsumers) sourceDispatcher).getCursor().getCursorProperties() + .keySet().containsAll(bucketKeys)); + } + @Test public void testUnsubscribe() throws Exception { From d84e271fd49157cd7b7eeeb199b947140641d1f8 Mon Sep 17 00:00:00 2001 From: SongOf <46475785+SongOf@users.noreply.github.com> Date: Mon, 20 Jul 2026 17:27:03 +0800 Subject: [PATCH 119/213] [fix][meta] Fix NPE in shouldIgnoreEvent when MetadataEvent options is null (#26200) Co-authored-by: maxlisongsong (cherry picked from commit 0ca94f57787457eb602d4518725502dac0dfc170) --- .../org/apache/pulsar/metadata/impl/AbstractMetadataStore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java index d118a792e2f23..92b1b9e2b7f80 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java @@ -228,7 +228,7 @@ protected boolean shouldIgnoreEvent(MetadataEvent event, GetResult existingValue } // ignore event if metadata is ephemeral or // sequential - if (options.contains(CreateOption.Ephemeral) || event.getOptions().contains(CreateOption.Sequential)) { + if (options.contains(CreateOption.Ephemeral) || options.contains(CreateOption.Sequential)) { return true; } // ignore the event if event occurred before the From 6fe3a236f85fd7a81e7ed9f7b3ce48f1b3852d06 Mon Sep 17 00:00:00 2001 From: Dream95 <864197662@qq.com> Date: Tue, 21 Jul 2026 22:37:35 +0800 Subject: [PATCH 120/213] [fix][fn] Return inputSpecs consumerProperties in function GET info (#26217) Signed-off-by: Dream95 (cherry picked from commit 90cba329a1fb4c306fc04d84f792da34ee7906c9) --- .../functions/utils/FunctionConfigUtils.java | 1 + .../utils/FunctionConfigUtilsTest.java | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java index 45fb4c1cb1ee7..33e2d795b16ad 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java @@ -411,6 +411,7 @@ public static FunctionConfig convertFromDetails(FunctionDetails functionDetails) } consumerConfig.setRegexPattern(input.getValue().getIsRegexPattern()); consumerConfig.setSchemaProperties(input.getValue().getSchemaPropertiesMap()); + consumerConfig.setConsumerProperties(input.getValue().getConsumerPropertiesMap()); consumerConfig.setPoolMessages(input.getValue().getPoolMessages()); consumerConfigMap.put(input.getKey(), consumerConfig); } diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java index 4679553da38d6..c3904ec9ef152 100644 --- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java +++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java @@ -691,6 +691,27 @@ public void testPoolMessages() { assertTrue(convertedConfig.getInputSpecs().get("test-input").isPoolMessages()); } + @Test + public void testConsumerProperties() { + FunctionConfig functionConfig = createFunctionConfig(); + + Map consumerProperties = new HashMap<>(); + consumerProperties.put("consumerName", "window-consumer"); + Map inputSpecs = new HashMap<>(); + inputSpecs.put("test-input", ConsumerConfig.builder() + .consumerProperties(consumerProperties) + .build()); + functionConfig.setInputSpecs(inputSpecs); + + FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig); + assertEquals(functionDetails.getSource().getInputSpecsMap().get("test-input").getConsumerPropertiesMap(), + consumerProperties); + + FunctionConfig convertedConfig = FunctionConfigUtils.convertFromDetails(functionDetails); + assertEquals(convertedConfig.getInputSpecs().get("test-input").getConsumerProperties(), + consumerProperties); + } + @Test public void testConvertProducerSpecToProducerConfigAndBackToProducerSpec() { // given From f82268a266f4170427099e1ab52baceae5ff74a8 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 22 Jul 2026 13:06:43 +0300 Subject: [PATCH 121/213] [fix][broker][branch-4.2] Fix admin API HTTP 400 FAIL_ON_TRAILING_TOKENS when a broker interceptor is loaded (#26223) (cherry picked from commit 7a172adc2ede1e37819d9351cfde86d224c9e45c) --- .../pulsar/broker/web/RequestWrapper.java | 69 +++++++++++++------ 1 file changed, 48 insertions(+), 21 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/RequestWrapper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/RequestWrapper.java index afebbd276eba8..0530f46abbb51 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/RequestWrapper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/RequestWrapper.java @@ -36,36 +36,63 @@ */ public class RequestWrapper extends HttpServletRequestWrapper { - private final byte[] body; + private byte[] body; + private ServletInputStream inputStream; public RequestWrapper(HttpServletRequest request) throws IOException { super(request); - body = IOUtils.toByteArray(new InputStreamReader(request.getInputStream()), Charset.defaultCharset()); + } + + /** + * Buffers the request body on first access. Buffering is lazy on purpose: an interceptor that + * does not read the body (the common case) must not cause the body to be buffered at all, so the + * underlying request stream is consumed only once, by the downstream resource (Jersey). The read + * is bounded to Content-Length so it never reads past the request body. + */ + private byte[] body() throws IOException { + if (body == null) { + HttpServletRequest request = (HttpServletRequest) getRequest(); + int contentLength = request.getContentLength(); + if (contentLength >= 0) { + body = IOUtils.toByteArray(request.getInputStream(), contentLength); + } else { + body = IOUtils.toByteArray(request.getInputStream()); + } + } + return body; } @Override public ServletInputStream getInputStream() throws IOException { - final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(body); - return new ServletInputStream() { - @Override - public boolean isFinished() { - return false; - } + if (inputStream == null) { + // Return a single, stable stream over the buffered body. Repeated getInputStream() calls + // must return the same stream (per the Servlet contract); returning a fresh stream each + // time lets a reader that re-fetches the stream after EOF read the body's first byte + // again, surfacing as a spurious "Trailing token" error. + final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(body()); + inputStream = new ServletInputStream() { + @Override + public boolean isFinished() { + return byteArrayInputStream.available() == 0; + } - @Override - public boolean isReady() { - return true; - } + @Override + public boolean isReady() { + return true; + } - @Override - public void setReadListener(ReadListener readListener) { + @Override + public void setReadListener(ReadListener readListener) { - } + } - public int read() { - return byteArrayInputStream.read(); - } - }; + @Override + public int read() { + return byteArrayInputStream.read(); + } + }; + } + return inputStream; } @Override @@ -74,7 +101,7 @@ public BufferedReader getReader() throws IOException { } //Use this method to read the request body N times - public byte[] getBody() { - return this.body; + public byte[] getBody() throws IOException { + return body(); } } From 3bbe882c9f4a25fc4a5adc4055bb6ceb95c8b53a Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 22 Jul 2026 13:05:40 +0300 Subject: [PATCH 122/213] [fix][ci][branch-4.0] Fix OpenTelemetrySanityTest after Otel library upgrade --- .../tests/integration/metrics/OpenTelemetrySanityTest.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/metrics/OpenTelemetrySanityTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/metrics/OpenTelemetrySanityTest.java index 4e6847d17235e..6f909c2fd5b7d 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/metrics/OpenTelemetrySanityTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/metrics/OpenTelemetrySanityTest.java @@ -45,6 +45,7 @@ public class OpenTelemetrySanityTest { // Validate that the OpenTelemetry metrics can be exported to a remote OpenTelemetry collector. @Test(timeOut = 360_000) + @SuppressWarnings("unchecked") public void testOpenTelemetryMetricsOtlpExport() throws Exception { var clusterName = "testOpenTelemetryMetrics-" + UUID.randomUUID(); var openTelemetryCollectorContainer = new OpenTelemetryCollectorContainer(clusterName); @@ -71,7 +72,7 @@ public void testOpenTelemetryMetricsOtlpExport() throws Exception { // TODO: Validate cluster name and service version are present once // https://github.com/open-telemetry/opentelemetry-java/issues/6108 is solved. - var metricName = "queueSize_ratio"; // Sent automatically by the OpenTelemetry SDK. + var metricName = "jvm_cpu_count"; // Configured by the OpenTelemetryService. waitAtMost(90, TimeUnit.SECONDS).ignoreExceptions().pollInterval(1, TimeUnit.SECONDS).until(() -> { var metrics = getMetricsFromPrometheus( openTelemetryCollectorContainer, OpenTelemetryCollectorContainer.PROMETHEUS_EXPORTER_PORT); @@ -95,6 +96,7 @@ public void testOpenTelemetryMetricsOtlpExport() throws Exception { * https://github.com/open-telemetry/opentelemetry-java/blob/main/sdk-extensions/autoconfigure/README.md#prometheus-exporter */ @Test(timeOut = 360_000) + @SuppressWarnings("unchecked") public void testOpenTelemetryMetricsPrometheusExport() throws Exception { var prometheusExporterPort = 9464; var clusterName = "testOpenTelemetryMetrics-" + UUID.randomUUID(); @@ -158,6 +160,7 @@ private static PrometheusMetricsClient.Metrics getMetricsFromPrometheus(ChaosCon return client.getMetrics(); } + @SuppressWarnings("unchecked") private static Map getOpenTelemetryProps(String exporter, Pair ... extraProps) { var defaultProps = Map.of( "OTEL_SDK_DISABLED", "false", From c493dffbbaea01912922a8548ad519dad372b8e2 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 22 Jul 2026 18:01:03 +0300 Subject: [PATCH 123/213] Fix binary license --- distribution/server/src/assemble/LICENSE.bin.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 37c1476e915ea..c1d080545ad1f 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -278,6 +278,7 @@ The Apache Software License, Version 2.0 - io.swagger-swagger-annotations-1.6.2.jar - io.swagger-swagger-core-1.6.2.jar - io.swagger-swagger-models-1.6.2.jar + * slog -- io.github.merlimat.slog-slog-0.9.5.jar * DataSketches - com.yahoo.datasketches-memory-0.8.3.jar - com.yahoo.datasketches-sketches-core-0.8.3.jar From bad0e295153ea9969382e31d99663b5768d6b92a Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 22 Jul 2026 18:12:35 +0300 Subject: [PATCH 124/213] [fix][misc][branch-4.0] Make log4j pattern compatible with slog which got pulled in by Oxia client upgrade --- conf/functions_log4j2.xml | 6 +++--- conf/log4j2.yaml | 6 +++--- microbench/src/main/resources/log4j2.xml | 2 +- pulsar-functions/localrun/src/main/resources/log4j2.xml | 2 +- .../runtime-all/src/main/resources/java_instance_log4j2.xml | 6 +++--- .../src/main/resources/kubernetes_instance_log4j2.xml | 2 +- tiered-storage/jcloud/src/test/resources/log4j2-test.yml | 2 +- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/conf/functions_log4j2.xml b/conf/functions_log4j2.xml index fd4042e82e82f..b4c36867e2d42 100644 --- a/conf/functions_log4j2.xml +++ b/conf/functions_log4j2.xml @@ -40,7 +40,7 @@ Console SYSTEM_OUT - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg %X%n @@ -49,7 +49,7 @@ ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz true - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg %X%n @@ -82,7 +82,7 @@ ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz true - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg %X%n diff --git a/conf/log4j2.yaml b/conf/log4j2.yaml index 2571c2d9ab9e1..52b5e92830851 100644 --- a/conf/log4j2.yaml +++ b/conf/log4j2.yaml @@ -58,7 +58,7 @@ Configuration: - name: Console target: SYSTEM_OUT PatternLayout: - Pattern: "%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n" + Pattern: "%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg %X%n" - name: ConsoleJson target: SYSTEM_OUT JsonTemplateLayout: @@ -71,7 +71,7 @@ Configuration: filePattern: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}-%d{MM-dd-yyyy}-%i.log.gz" immediateFlush: ${sys:pulsar.log.immediateFlush} PatternLayout: - Pattern: "%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n" + Pattern: "%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg %X%n" Policies: TimeBasedTriggeringPolicy: interval: 1 @@ -109,7 +109,7 @@ Configuration: fileName : "${sys:pulsar.log.dir}/functions/${ctx:function}/${ctx:functionname}-${ctx:instance}.log" filePattern : "${sys:pulsar.log.dir}/functions/${sys:pulsar.log.file}-${ctx:instance}-%d{MM-dd-yyyy}-%i.log.gz" PatternLayout: - Pattern: "%d{ABSOLUTE} %level{length=5} [%thread] [instance: %X{instance}] %logger{1} - %msg%n" + Pattern: "%d{ABSOLUTE} %level{length=5} [%thread] [instance: %X{instance}] %logger{1} - %msg %X%n" Policies: TimeBasedTriggeringPolicy: interval: 1 diff --git a/microbench/src/main/resources/log4j2.xml b/microbench/src/main/resources/log4j2.xml index 7ec5ed8169a66..64b03ab62f43c 100644 --- a/microbench/src/main/resources/log4j2.xml +++ b/microbench/src/main/resources/log4j2.xml @@ -22,7 +22,7 @@ - + diff --git a/pulsar-functions/localrun/src/main/resources/log4j2.xml b/pulsar-functions/localrun/src/main/resources/log4j2.xml index a7714c3f48835..d29e798e4797a 100644 --- a/pulsar-functions/localrun/src/main/resources/log4j2.xml +++ b/pulsar-functions/localrun/src/main/resources/log4j2.xml @@ -22,7 +22,7 @@ - + diff --git a/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml b/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml index 190d9be92940b..c6e4b8eeb86eb 100644 --- a/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml +++ b/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml @@ -40,7 +40,7 @@ Console SYSTEM_OUT - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg %X%n @@ -49,7 +49,7 @@ ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz true - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg %X%n @@ -82,7 +82,7 @@ ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz true - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg %X%n diff --git a/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml b/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml index f86d03e41793f..d245fe7714fa1 100644 --- a/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml +++ b/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml @@ -36,7 +36,7 @@ Console SYSTEM_OUT - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg %X%n diff --git a/tiered-storage/jcloud/src/test/resources/log4j2-test.yml b/tiered-storage/jcloud/src/test/resources/log4j2-test.yml index f5ee5c9a53dd7..77b3baa97a970 100644 --- a/tiered-storage/jcloud/src/test/resources/log4j2-test.yml +++ b/tiered-storage/jcloud/src/test/resources/log4j2-test.yml @@ -33,7 +33,7 @@ Configuration: name: STDOUT target: SYSTEM_OUT PatternLayout: - Pattern: "%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n" + Pattern: "%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg %X%n" File: name: File fileName: ${filename} From 37d178b5a956f74dd89408cd751df87c5f569359 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Sun, 19 Jul 2026 00:00:06 +0800 Subject: [PATCH 125/213] [fix][test] Fix flaky test `testCompactionPriority ` (#26198) (cherry picked from commit 842cfac9349d2992f6106b9627d9b0bc2662a0e5) --- .../pulsar/broker/admin/AdminApi2Test.java | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java index fe8926142a6e8..00c5f076d4c5c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java @@ -27,6 +27,7 @@ import static org.apache.pulsar.common.policies.data.NamespaceIsolationPolicyUnloadScope.none; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -40,7 +41,6 @@ import static org.testng.Assert.fail; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import java.lang.reflect.Field; import java.net.URI; import java.net.URL; import java.net.http.HttpClient; @@ -70,7 +70,6 @@ import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.AsyncCallbacks; -import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; @@ -3240,11 +3239,14 @@ public void testCompactionPriority() throws Exception { final String namespace = newUniqueName(defaultTenant + "/ns"); admin.namespaces().createNamespace(namespace, Set.of("test")); final String topic = "persistent://" + namespace + "/topic" + UUID.randomUUID(); - pulsarClient.newProducer().topic(topic).create().close(); + @Cleanup + Producer producer = pulsarClient.newProducer().topic(topic).create(); + producer.send("message".getBytes()); TopicName topicName = TopicName.get(topic); PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService() .getTopicIfExists(topic).get().get(); PersistentTopic mockTopic = spy(persistentTopic); + doNothing().when(mockTopic).triggerCompaction(); mockTopic.checkCompaction(); // Disabled by default verify(mockTopic, times(0)).triggerCompaction(); @@ -3252,27 +3254,31 @@ public void testCompactionPriority() throws Exception { admin.namespaces().setCompactionThreshold(namespace, 1); Awaitility.await().untilAsserted(() -> assertNotNull(admin.namespaces().getCompactionThreshold(namespace))); - ManagedLedger managedLedger = persistentTopic.getManagedLedger(); - Field field = managedLedger.getClass().getDeclaredField("totalSize"); - field.setAccessible(true); - field.setLong(managedLedger, 1000L); + Awaitility.await().untilAsserted(() -> assertTrue(persistentTopic.isCompactionEnabled())); - mockTopic.checkCompaction(); - verify(mockTopic, times(1)).triggerCompaction(); + Awaitility.await().untilAsserted(() -> { + mockTopic.checkCompaction(); + verify(mockTopic, times(1)).triggerCompaction(); + }); //Set topic-level policy admin.topics().setCompactionThreshold(topic, 0); Awaitility.await().untilAsserted(() -> assertNotNull(admin.topics().getCompactionThreshold(topic))); + Awaitility.await().untilAsserted(() -> assertFalse(persistentTopic.isCompactionEnabled())); mockTopic.checkCompaction(); verify(mockTopic, times(1)).triggerCompaction(); // Remove topic-level policy admin.topics().removeCompactionThreshold(topic); Awaitility.await().untilAsserted(() -> assertNull(admin.topics().getCompactionThreshold(topic))); - mockTopic.checkCompaction(); - verify(mockTopic, times(2)).triggerCompaction(); + Awaitility.await().untilAsserted(() -> assertTrue(persistentTopic.isCompactionEnabled())); + Awaitility.await().untilAsserted(() -> { + mockTopic.checkCompaction(); + verify(mockTopic, times(2)).triggerCompaction(); + }); // Remove namespace-level policy admin.namespaces().removeCompactionThreshold(namespace); Awaitility.await().untilAsserted(() -> assertNull(admin.namespaces().getCompactionThreshold(namespace))); + Awaitility.await().untilAsserted(() -> assertFalse(persistentTopic.isCompactionEnabled())); mockTopic.checkCompaction(); verify(mockTopic, times(2)).triggerCompaction(); } From efe0c120fceb76269b8154b8959bbef29f41963b Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 22 Jul 2026 19:06:40 +0300 Subject: [PATCH 126/213] [improve][misc][branch-4.0] Add CustomLog config for slog --- lombok.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lombok.config b/lombok.config index 1be7aa1e6bc18..19d2ae53117a4 100644 --- a/lombok.config +++ b/lombok.config @@ -21,5 +21,5 @@ # see https://projectlombok.org/features/configuration for reference config.stopBubbling = true - +lombok.log.custom.declaration = io.github.merlimat.slog.Logger io.github.merlimat.slog.Logger.get(TYPE) From 8147cd2821a5fcec5233b44882db915c56682757 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 23 Jul 2026 16:14:43 +0800 Subject: [PATCH 127/213] [improve][broker][branch-4.0] Trace the asynchronous tasks in logs when loading topics (#26163) (#26224) Signed-off-by: Zixuan Liu --- .../pulsar/broker/service/BrokerService.java | 107 +++++++------ .../broker/service/TopicLoadingContext.java | 36 ++--- .../pulsar/common/util/LatencyTracer.java | 110 ++++++++++++++ .../pulsar/common/util/LatencyTracerTest.java | 143 ++++++++++++++++++ 4 files changed, 316 insertions(+), 80 deletions(-) create mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/util/LatencyTracer.java create mode 100644 pulsar-common/src/test/java/org/apache/pulsar/common/util/LatencyTracerTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 1f7983306dc52..5fabf6c10e3e4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -1216,32 +1216,35 @@ public CompletableFuture> getTopic(TopicLoadingContext context) return FutureUtil.failedFuture(new NotAllowedException( "Broker is unable to load persistent topic")); } + if (context.getTopicFuture() == null) { + final var timeoutSeconds = pulsar.getConfiguration().getTopicLoadTimeoutSeconds(); final CompletableFuture> topicFuture = FutureUtil.createFutureWithTimeout( - Duration.ofSeconds(pulsar.getConfiguration().getTopicLoadTimeoutSeconds()), executor(), + Duration.ofSeconds(timeoutSeconds), executor(), () -> FAILED_TO_LOAD_TOPIC_TIMEOUT_EXCEPTION); context.setTopicFuture(topicFuture); } var topicFuture = context.getTopicFuture(); topicFuture.exceptionally(t -> { - final var now = System.nanoTime(); + final var latency = context.traceAndGetLatency("fail").description(); if (FutureUtil.unwrapCompletionException(t) instanceof TimeoutException) { - log.warn("Failed to load {} after {} ms", topicName, context.latencyMs(now)); + log.warn("Failed to load {} within {} s (latency: {})", topicName, timeoutSeconds, latency); } else { - log.warn("Failed to load {} after {} ms", topicName, context.latencyString(now), t); + log.warn("Failed to load {} (latency: {})", topicName, latency, t); } pulsarStats.recordTopicLoadFailed(); return Optional.empty(); }); - checkNonPartitionedTopicExists(topicName).thenAccept(exists -> { + context.trace("topic exists", checkNonPartitionedTopicExists(topicName)).thenAccept(exists -> { if (!exists && !createIfMissing) { topicFuture.complete(Optional.empty()); return; } // The topic level policies are not needed now, but the meaning of calling // "getTopicPoliciesBypassSystemTopic" will wait for system topic policies initialization. - getTopicPoliciesBypassSystemTopic(topicName, TopicPoliciesService.GetType.LOCAL_ONLY) - .thenRun(() -> { + final var systemTopicLoadFuture = context.trace("system topic", + getTopicPoliciesBypassSystemTopic(topicName, TopicPoliciesService.GetType.LOCAL_ONLY)); + systemTopicLoadFuture.thenRun(() -> { final var inserted = new MutableBoolean(false); final var cachedFuture = topics.computeIfAbsent(topicName.toString(), ___ -> { inserted.setTrue(); @@ -1262,7 +1265,7 @@ public CompletableFuture> getTopic(TopicLoadingContext context) // checked if the `topics` cache includes this topic before, so the latency is not the // actual loading latency that should not be recorded in metrics. log.info("[{}] Finished loading from other concurrent loading task (latency: {})", - topicName, context.latencyString(System.nanoTime())); + topicName, context.getLatency().description()); cachedFuture.whenComplete((optTopic, e) -> { if (e == null) { topicFuture.complete(optTopic); @@ -1272,15 +1275,14 @@ public CompletableFuture> getTopic(TopicLoadingContext context) }); } }).exceptionally(e -> { - pulsar.getExecutor().execute(() -> topics.remove(topicName.toString(), topicFuture)); - final Throwable rc = FutureUtil.unwrapCompletionException(e); - final String errorInfo = String.format("Topic creation encountered an exception by initialize" - + " topic policies service. topic_name=%s error_message=%s", topicName, - rc.getMessage()); - log.error(errorInfo, rc); - topicFuture.completeExceptionally(rc); + log.warn("[{}] Topic creation encountered an exception" + + " by initialize topic policies service", topicName); + failTopicFuture(topicName.toString(), topicFuture, e); return null; }); + }).exceptionally(e -> { + failTopicFuture(topicName.toString(), topicFuture, e); + return null; }); return topicFuture; } else { @@ -1326,6 +1328,14 @@ public CompletableFuture> getTopic(TopicLoadingContext context) } } + private void failTopicFuture(String topic, CompletableFuture> topicFuture, Throwable throwable) { + pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); + final Throwable rc = FutureUtil.unwrapCompletionException(throwable); + // It will trigger the logging for exception and traced latencies in topicFuture's exceptionally callback, so + // we don't need to add an extra log before it. + topicFuture.completeExceptionally(rc); + } + private CompletableFuture> getTopicPoliciesBypassSystemTopic(@NonNull TopicName topicName, TopicPoliciesService.GetType type) { if (ExtensibleLoadManagerImpl.isInternalTopic(topicName.toString())) { @@ -1802,7 +1812,8 @@ public PulsarAdmin getClusterPulsarAdmin(String cluster, Optional c protected CompletableFuture> loadOrCreatePersistentTopic(TopicLoadingContext context) { final var topic = context.getTopicName().toString(); final var topicFuture = context.getTopicFuture(); - checkTopicNsOwnership(topic) + final var ownedFuture = checkTopicNsOwnership(topic); + context.trace("ownership", ownedFuture) .thenRun(() -> { final Semaphore topicLoadSemaphore = topicLoadRequestSemaphore.get(); @@ -1814,8 +1825,7 @@ protected CompletableFuture> loadOrCreatePersistentTopic(TopicLo // do not recreate topic if topic is already migrated and deleted by broker // so, avoid creating a new topic if migration is already started if (ex != null && (ex.getCause() instanceof TopicMigratedException)) { - pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); - topicFuture.completeExceptionally(ex.getCause()); + failTopicFuture(topic, topicFuture, ex); return null; } createPendingLoadTopic(); @@ -1828,8 +1838,7 @@ protected CompletableFuture> loadOrCreatePersistentTopic(TopicLo } } }).exceptionally(ex -> { - pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); - topicFuture.completeExceptionally(ex.getCause()); + failTopicFuture(topic, topicFuture, ex); return null; }); @@ -1869,7 +1878,7 @@ private void checkOwnershipAndCreatePersistentTopic(TopicLoadingContext context) TopicName topicName = context.getTopicName(); final var topic = topicName.toString(); final var topicFuture = context.getTopicFuture(); - checkTopicNsOwnership(topic).thenRun(() -> { + context.trace("2nd ownership", checkTopicNsOwnership(topic)).thenRun(() -> { CompletableFuture> propertiesFuture; if (context.getProperties() == null) { //Read properties from storage when loading topic. @@ -1877,19 +1886,16 @@ private void checkOwnershipAndCreatePersistentTopic(TopicLoadingContext context) } else { propertiesFuture = CompletableFuture.completedFuture(context.getProperties()); } - propertiesFuture.thenAccept(finalProperties -> { + context.trace("properties", propertiesFuture).thenAccept(finalProperties -> { context.setProperties(finalProperties); //TODO add topicName in properties? createPersistentTopic0(context); }).exceptionally(throwable -> { - log.warn("[{}] Read topic property failed", topic, throwable); - pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); - topicFuture.completeExceptionally(throwable); + failTopicFuture(topic, topicFuture, throwable); return null; }); }).exceptionally(e -> { - pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); - topicFuture.completeExceptionally(e.getCause()); + failTopicFuture(topic, topicFuture, e); return null; }); } @@ -1902,10 +1908,8 @@ public void createPersistentTopic0(TopicLoadingContext context) { final var createIfMissing = context.isCreateIfMissing(); if (isTransactionInternalName(topicName)) { - String msg = String.format("Can not create transaction system topic %s", topic); - log.warn(msg); - pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); - topicFuture.completeExceptionally(new NotAllowedException(msg)); + failTopicFuture(topic, topicFuture, new NotAllowedException("Can not create transaction system topic " + + topic)); return; } @@ -1922,6 +1926,7 @@ public void createPersistentTopic0(TopicLoadingContext context) { n.recycle(); return found; }), (managedLedgerConfig, exists) -> { + context.trace("ml-config"); if (isBrokerEntryMetadataEnabled() || isBrokerPayloadProcessorEnabled()) { // init managedLedger interceptor Set interceptors = new HashSet<>(); @@ -1967,25 +1972,24 @@ public void createPersistentTopic0(TopicLoadingContext context) { @Override public void openLedgerComplete(ManagedLedger ledger, Object ctx) { try { + context.trace("open-ml"); PersistentTopic persistentTopic = isSystemTopic(topic) ? new SystemTopic(topic, ledger, BrokerService.this) : newTopic(topic, ledger, BrokerService.this, PersistentTopic.class); persistentTopic.setCreateFuture(topicFuture); - persistentTopic - .initialize() - .thenCompose(__ -> persistentTopic.preCreateSubscriptionForCompactionIfNeeded()) - .thenCompose(__ -> persistentTopic.checkReplication()) - .thenCompose(v -> { - // Also check dedup status - return persistentTopic.checkDeduplicationStatus(); - }) + context.trace("init", persistentTopic.initialize()) + .thenCompose(__ -> context.trace("pre-create compacted sub", + persistentTopic.preCreateSubscriptionForCompactionIfNeeded())) + .thenCompose(__ -> context.trace("replication", + persistentTopic.checkReplication())) + .thenCompose(v -> context.trace("deduplication", + persistentTopic.checkDeduplicationStatus())) .thenRun(() -> { - long nowInNanos = System.nanoTime(); - long topicLoadLatencyMs = context.latencyMs(nowInNanos); + final var latency = context.traceAndGetLatency("done"); log.info("Created topic {} - dedup is {} (latency: {})", topic, persistentTopic.isDeduplicationEnabled() ? "enabled" : "disabled", - context.latencyString(nowInNanos)); - pulsarStats.recordTopicLoadTimeValue(topic, topicLoadLatencyMs); + latency.description()); + pulsarStats.recordTopicLoadTimeValue(topic, latency.elapsedInMillis()); if (createEventBuild != null) { createEventBuild .stage(EventStage.SUCCESS) @@ -2043,9 +2047,7 @@ public void openLedgerComplete(ManagedLedger ledger, Object ctx) { .error(e) .dispatch(); } - log.warn("Failed to create topic {}: {}", topic, e.getMessage()); - pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); - topicFuture.completeExceptionally(e); + failTopicFuture(topic, topicFuture, e); } } @@ -2056,9 +2058,7 @@ public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); topicFuture.complete(Optional.empty()); } else { - log.warn("Failed to create topic {}", topic, exception); - pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); - topicFuture.completeExceptionally(new PersistenceException(exception)); + failTopicFuture(topic, topicFuture, new PersistenceException(exception)); } } }, () -> isTopicNsOwnedByBrokerAsync(topicName), null); @@ -2066,12 +2066,9 @@ public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { }).exceptionally((exception) -> { boolean migrationFailure = exception.getCause() instanceof TopicMigratedException; String msg = migrationFailure ? "Topic is already migrated" : - "Failed to get topic configuration:"; - log.warn("[{}] {} {}", topic, msg, exception.getMessage(), exception); - // remove topic from topics-map in different thread to avoid possible deadlock if - // createPersistentTopic-thread only tries to handle this future-result - pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture)); - topicFuture.completeExceptionally(exception); + "Failed to get topic configuration"; + log.warn("[{}] {}", topic, msg); + failTopicFuture(topic, topicFuture, exception); return null; }); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicLoadingContext.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicLoadingContext.java index 645cf18d8c733..dcba755ae0e07 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicLoadingContext.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicLoadingContext.java @@ -21,19 +21,16 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; -import lombok.Builder; +import java.util.concurrent.ConcurrentLinkedQueue; import lombok.Getter; import lombok.Setter; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.util.LatencyTracer; import org.jspecify.annotations.Nullable; @Builder -public class TopicLoadingContext { +public class TopicLoadingContext extends LatencyTracer { - private static final String EXAMPLE_LATENCY_OUTPUTS = "1234 ms (queued: 567)"; - - private final long startNs = System.nanoTime(); @Getter private final TopicName topicName; @Getter @@ -44,9 +41,7 @@ public class TopicLoadingContext { private CompletableFuture> topicFuture; @Getter @Setter - @Nullable - private Map properties; - private long polledFromQueueNs = -1L; + @Nullable private Map properties; @Getter @Setter @@ -55,21 +50,12 @@ public class TopicLoadingContext { @Setter private String proxyVersion; - public void polledFromQueue() { - polledFromQueueNs = System.nanoTime(); - } - - public long latencyMs(long nowInNanos) { - return TimeUnit.NANOSECONDS.toMillis(nowInNanos - startNs); - } - - public String latencyString(long nowInNanos) { - final var builder = new StringBuilder(EXAMPLE_LATENCY_OUTPUTS.length()); - builder.append(latencyMs(nowInNanos)); - builder.append(" ms"); - if (polledFromQueueNs >= 0) { - builder.append(" (queued: ").append(latencyMs(polledFromQueueNs)).append(")"); - } - return builder.toString(); + public TopicLoadingContext(TopicName topicName, boolean createIfMissing, + CompletableFuture> topicFuture) { + // The topic loading could be ended asynchronously by a timeout event, so we need a thread safe queue here + super(new ConcurrentLinkedQueue<>(), System::nanoTime); + this.topicName = topicName; + this.createIfMissing = createIfMissing; + this.topicFuture = topicFuture; } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/LatencyTracer.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/LatencyTracer.java new file mode 100644 index 0000000000000..afc335a16e4bf --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/LatencyTracer.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util; + +import java.util.ArrayList; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.experimental.Accessors; + +public class LatencyTracer { + + private final Queue timepoints; + private final NanoTimeSupplier nanoTimeSupplier; + private final long startNs; + + public LatencyTracer(Queue timepoints, NanoTimeSupplier nanoTimeSupplier) { + this.timepoints = timepoints; + this.nanoTimeSupplier = nanoTimeSupplier; + this.startNs = nanoTimeSupplier.getNanos(); + } + + public CompletableFuture trace(String message, CompletableFuture future) { + if (future.isDone()) { + return future; + } + return future.whenComplete((__, ___) -> trace(message)); + } + + public void trace(String action) { + timepoints.add(new Timepoint(action, nanoTimeSupplier.getNanos())); + } + + public Snapshot getLatency() { + final List timepoints = new ArrayList<>(this.timepoints); + if (timepoints.isEmpty()) { + return new Snapshot(startNs, 0, "total: 0 ms"); + } + final StringBuilder sb = new StringBuilder(); + final long totalLatencyNs = TimeUnit.NANOSECONDS.toMillis(timepoints.get(timepoints.size() - 1).timeInNanos + - startNs); + sb.append("total: ").append(totalLatencyNs).append(" ms"); + long prevNs = startNs; + for (final Timepoint tp : timepoints) { + sb.append(", ").append(tp.name).append(": "); + long latencyMs = TimeUnit.NANOSECONDS.toMillis(tp.timeInNanos - prevNs); + if (latencyMs > 0) { + sb.append(latencyMs).append(" ms"); + } else { + sb.append(TimeUnit.NANOSECONDS.toMicros(tp.timeInNanos - prevNs)).append(" us"); + } + prevNs = tp.timeInNanos; + } + return new Snapshot(prevNs, totalLatencyNs, sb.toString()); + } + + public Snapshot traceAndGetLatency(String action) { + trace(action); + return getLatency(); + } + + public interface NanoTimeSupplier { + + long getNanos(); + } + + @Accessors(fluent = true) + @Getter + @RequiredArgsConstructor + public static final class Timepoint { + + private final String name; + private final long timeInNanos; + } + + /** + * A snapshot of the latency tracer at a given moment. + * `endTimeInNanos` the latest traced timestamp in nanoseconds + * `elapsedInMillis` the elapsed time in milliseconds from when the tracer was created + * `description` the detailed description for traced latencies, e.g. "total: 100 ms, A: 60 ms, B: 40 ms" + */ + @Accessors(fluent = true) + @Getter + @RequiredArgsConstructor + public static final class Snapshot { + + private final long endTimeInNanos; + private final long elapsedInMillis; + private final String description; + } +} diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/LatencyTracerTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/LatencyTracerTest.java new file mode 100644 index 0000000000000..d3d638d096e57 --- /dev/null +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/LatencyTracerTest.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNotSame; +import static org.testng.Assert.assertSame; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; +import java.util.LinkedList; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.testng.annotations.Test; + +@Test(groups = "utils") +public class LatencyTracerTest { + + private static LatencyTracer.NanoTimeSupplier testNanoTimeSupplier(long... nanoTimes) { + return new LatencyTracer.NanoTimeSupplier() { + final LinkedList nanoTimesQueue = new LinkedList<>(); + + { + for (long nanoTime : nanoTimes) { + nanoTimesQueue.add(nanoTime); + } + } + + @Override + public long getNanos() { + final Long nanos = nanoTimesQueue.poll(); + assertNotNull(nanos); + return nanos; + } + }; + } + + @Test + public void testMulti() { + LatencyTracer tracer = new LatencyTracer(new LinkedList<>(), testNanoTimeSupplier(10_000_000L, 30_000_000L, + 70_000_000L, 80_000_000L)); + tracer.trace("A"); + tracer.trace("B"); + LatencyTracer.Snapshot snapshot = tracer.getLatency(); + assertEquals(snapshot.description(), "total: 60 ms, A: 20 ms, B: 40 ms"); + assertEquals(snapshot.elapsedInMillis(), 60); + + tracer.trace("C"); + snapshot = tracer.getLatency(); + assertEquals(snapshot.description(), "total: 70 ms, A: 20 ms, B: 40 ms, C: 10 ms"); + assertEquals(snapshot.elapsedInMillis(), 70); + } + + @Test + public void testEmpty() { + LatencyTracer tracer = new LatencyTracer(new LinkedList<>(), testNanoTimeSupplier(0L, 20_000_000L)); + LatencyTracer.Snapshot snapshot = tracer.getLatency(); + assertEquals(snapshot.description(), "total: 0 ms"); + assertEquals(snapshot.elapsedInMillis(), 0); + } + + @Test + public void testZeroMs() { + LatencyTracer tracer = new LatencyTracer(new LinkedList<>(), testNanoTimeSupplier(0L, 999_999L, 2_000_000L, + 2_100_000L)); + tracer.trace("A"); + tracer.trace("B"); + tracer.trace("C"); + LatencyTracer.Snapshot snapshot = tracer.getLatency(); + assertEquals(snapshot.description(), "total: 2 ms, A: 999 us, B: 1 ms, C: 100 us"); + assertEquals(snapshot.elapsedInMillis(), 2); + } + + @Test + public void testTraceFuture() throws Exception { + LatencyTracer tracer = new LatencyTracer(new LinkedList<>(), System::nanoTime); + final CompletableFuture future = CompletableFuture.completedFuture(100); + assertSame(tracer.trace("A", future), future); + final String latency = tracer.getLatency().description(); + assertTrue(Pattern.compile("total: \\d+ ms").matcher(latency).matches(), latency); + + final CompletableFuture future2 = new CompletableFuture(); + delayedExecutor(500, TimeUnit.MILLISECONDS).execute(() -> future2.complete(1)); + final CompletableFuture tracedFuture = tracer.trace("B", future2); + assertNotSame(tracedFuture, future2); + assertEquals(tracedFuture.get(), 1); + final LatencyTracer.Snapshot snapshot = tracer.getLatency(); + Matcher m = Pattern.compile("total: \\d+ ms, B: (\\d+) ms").matcher(snapshot.description()); + assertTrue(m.matches(), snapshot.description()); + assertEquals(Long.parseLong(m.group(1)), snapshot.elapsedInMillis(), snapshot.description()); + assertTrue(snapshot.elapsedInMillis() >= 500, snapshot.description()); + } + + @Test + public void testTraceFailedFuture() throws Exception { + final LatencyTracer tracer = new LatencyTracer(new LinkedList<>(), testNanoTimeSupplier(1L)); + + final CompletableFuture future = new CompletableFuture(); + delayedExecutor(100, TimeUnit.MILLISECONDS).execute(() -> future.completeExceptionally( + new RuntimeException("failure"))); + + try { + tracer.trace("A", future).get(); + fail(); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof RuntimeException); + assertEquals(e.getCause().getMessage(), "failure"); + } + } + + // Replace it with `CompletableFuture.delayedExecutor` after upgraded to Java 9 or later + private static Executor delayedExecutor(long delay, TimeUnit unit) { + + return command -> new Thread(() -> { + try { + Thread.sleep(unit.toMillis(delay)); + command.run(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }).start(); + } +} From cfaa7fd24d51ff245f9f1aa958a25b74dbdb2016 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 23 Jul 2026 06:05:10 +0300 Subject: [PATCH 128/213] [improve][build] Upgrade slog to 0.10.0 (#26226) (cherry picked from commit f51864cd2a76276e4f30898f4e2781841268899d) --- distribution/server/src/assemble/LICENSE.bin.txt | 2 +- pom.xml | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index c1d080545ad1f..befe6ffc7aa16 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -278,7 +278,7 @@ The Apache Software License, Version 2.0 - io.swagger-swagger-annotations-1.6.2.jar - io.swagger-swagger-core-1.6.2.jar - io.swagger-swagger-models-1.6.2.jar - * slog -- io.github.merlimat.slog-slog-0.9.5.jar + * slog -- io.github.merlimat.slog-slog-0.10.0.jar * DataSketches - com.yahoo.datasketches-memory-0.8.3.jar - com.yahoo.datasketches-sketches-core-0.8.3.jar diff --git a/pom.xml b/pom.xml index 5842824ac4be6..7ce562c2ad039 100644 --- a/pom.xml +++ b/pom.xml @@ -299,6 +299,7 @@ flexible messaging model and an intuitive client API. 0.7.7 0.9.4 0.7.4 + 0.10.0 2.0 1.10.12 5.5.0 @@ -1316,6 +1317,11 @@ flexible messaging model and an intuitive client API. oxia-testcontainers ${oxia-testcontainers.version} + + io.github.merlimat.slog + slog + ${slog.version} + com.amazonaws From e8e76e3c89e3fbae24aa2ce0ee7438e7a2110075 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Sat, 24 Jan 2026 10:09:36 +0200 Subject: [PATCH 129/213] [improve][misc] Upgrade to Alpine 3.23 (#25180) (cherry picked from commit 2d04c93d329978db79598ae89554c5e230781104) --- docker/kinesis-producer-alpine/Dockerfile | 2 +- docker/pulsar/Dockerfile | 8 ++++---- src/update_python_protobuf_stubs_with_docker.sh | 4 ++-- .../tests/integration/containers/PulsarContainer.java | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docker/kinesis-producer-alpine/Dockerfile b/docker/kinesis-producer-alpine/Dockerfile index a33dc82f15866..5ec98ab3a766f 100644 --- a/docker/kinesis-producer-alpine/Dockerfile +++ b/docker/kinesis-producer-alpine/Dockerfile @@ -17,7 +17,7 @@ # under the License. # -ARG ALPINE_VERSION=3.21 +ARG ALPINE_VERSION=3.23 # Builds an Alpine image with kinesis_producer compiled for Alpine Linux / musl diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 67e2062a72147..8c0730e00e628 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -17,7 +17,7 @@ # under the License. # -ARG ALPINE_VERSION=3.21 +ARG ALPINE_VERSION=3.23 ARG IMAGE_JDK_MAJOR_VERSION=21 # First create a stage with just the Pulsar tarball and scripts @@ -73,9 +73,9 @@ RUN echo networkaddress.cache.negative.ttl=1 >> /opt/jvm/conf/security/java.secu FROM alpine:$ALPINE_VERSION AS snappy-java ARG SNAPPY_VERSION -RUN apk add git alpine-sdk util-linux cmake autoconf automake libtool openjdk17 maven curl bash tar -ENV JAVA_HOME=/usr -RUN curl -Ls https://github.com/xerial/snappy-java/archive/refs/tags/v$SNAPPY_VERSION.tar.gz | tar zxf - && cd snappy-java-$SNAPPY_VERSION && make clean-native native +RUN apk add git alpine-sdk util-linux cmake autoconf automake libtool openjdk17 maven curl bash tar findutils +ENV JAVA_HOME=/usr/lib/jvm/default-jvm +RUN curl -Ls https://github.com/xerial/snappy-java/archive/refs/tags/v$SNAPPY_VERSION.tar.gz | tar zxf - && cd snappy-java-$SNAPPY_VERSION && CMAKE_POLICY_VERSION_MINIMUM=3.5 make clean-native native ## Create final stage from Alpine image diff --git a/src/update_python_protobuf_stubs_with_docker.sh b/src/update_python_protobuf_stubs_with_docker.sh index ece0b5096d7c6..2e095418d1ba3 100755 --- a/src/update_python_protobuf_stubs_with_docker.sh +++ b/src/update_python_protobuf_stubs_with_docker.sh @@ -20,6 +20,6 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -# Create an inline docker container with alpine:3.21 base image and mount the current directory to it as the +# Create an inline docker container with alpine:3.23 base image and mount the current directory to it as the # working directory and run the script inside the container. -docker run --rm -v "$SCRIPT_DIR/..:/pulsar_src" -w /pulsar_src alpine:3.21 sh -c 'apk add --no-cache bash python3 && /pulsar_src/src/update_python_protobuf_stubs.sh' \ No newline at end of file +docker run --rm -v "$SCRIPT_DIR/..:/pulsar_src" -w /pulsar_src alpine:3.23 sh -c 'apk add --no-cache bash python3 && /pulsar_src/src/update_python_protobuf_stubs.sh' \ No newline at end of file diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PulsarContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PulsarContainer.java index 104590045ce86..c2beeab0ae84f 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PulsarContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PulsarContainer.java @@ -51,7 +51,7 @@ public abstract class PulsarContainer> exte public static final int BROKER_HTTP_PORT = 8080; public static final int BROKER_HTTPS_PORT = 8081; - public static final String ALPINE_IMAGE_NAME = "alpine:3.20"; + public static final String ALPINE_IMAGE_NAME = "alpine:3.23"; public static final String DEFAULT_IMAGE_NAME = System.getenv().getOrDefault("PULSAR_TEST_IMAGE_NAME", "apachepulsar/pulsar-test-latest-version:latest"); public static final String UPGRADE_TEST_IMAGE_NAME = System.getenv().getOrDefault("PULSAR_UPGRADE_TEST_IMAGE_NAME", From 9eb075f67ac52b56c74222bf96dd193fda146048 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 23 Jul 2026 01:00:19 +0300 Subject: [PATCH 130/213] [improve][build] Upgrade docker base image Alpine to 3.24 (#26225) (cherry picked from commit 045d628f3e7aafceda28268e21cbe9fc6eb81b5b) --- docker/kinesis-producer-alpine/Dockerfile | 2 +- docker/pulsar/Dockerfile | 2 +- src/update_python_protobuf_stubs_with_docker.sh | 4 ++-- .../pulsar/tests/integration/containers/PulsarContainer.java | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/kinesis-producer-alpine/Dockerfile b/docker/kinesis-producer-alpine/Dockerfile index 5ec98ab3a766f..a33bde59eca56 100644 --- a/docker/kinesis-producer-alpine/Dockerfile +++ b/docker/kinesis-producer-alpine/Dockerfile @@ -17,7 +17,7 @@ # under the License. # -ARG ALPINE_VERSION=3.23 +ARG ALPINE_VERSION=3.24 # Builds an Alpine image with kinesis_producer compiled for Alpine Linux / musl diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 8c0730e00e628..8a9d9e18dc841 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -17,7 +17,7 @@ # under the License. # -ARG ALPINE_VERSION=3.23 +ARG ALPINE_VERSION=3.24 ARG IMAGE_JDK_MAJOR_VERSION=21 # First create a stage with just the Pulsar tarball and scripts diff --git a/src/update_python_protobuf_stubs_with_docker.sh b/src/update_python_protobuf_stubs_with_docker.sh index 2e095418d1ba3..6bc3191f83141 100755 --- a/src/update_python_protobuf_stubs_with_docker.sh +++ b/src/update_python_protobuf_stubs_with_docker.sh @@ -20,6 +20,6 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -# Create an inline docker container with alpine:3.23 base image and mount the current directory to it as the +# Create an inline docker container with alpine:3.24 base image and mount the current directory to it as the # working directory and run the script inside the container. -docker run --rm -v "$SCRIPT_DIR/..:/pulsar_src" -w /pulsar_src alpine:3.23 sh -c 'apk add --no-cache bash python3 && /pulsar_src/src/update_python_protobuf_stubs.sh' \ No newline at end of file +docker run --rm -v "$SCRIPT_DIR/..:/pulsar_src" -w /pulsar_src alpine:3.24 sh -c 'apk add --no-cache bash python3 && /pulsar_src/src/update_python_protobuf_stubs.sh' \ No newline at end of file diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PulsarContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PulsarContainer.java index c2beeab0ae84f..cd62814cf3527 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PulsarContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PulsarContainer.java @@ -51,7 +51,7 @@ public abstract class PulsarContainer> exte public static final int BROKER_HTTP_PORT = 8080; public static final int BROKER_HTTPS_PORT = 8081; - public static final String ALPINE_IMAGE_NAME = "alpine:3.23"; + public static final String ALPINE_IMAGE_NAME = "alpine:3.24"; public static final String DEFAULT_IMAGE_NAME = System.getenv().getOrDefault("PULSAR_TEST_IMAGE_NAME", "apachepulsar/pulsar-test-latest-version:latest"); public static final String UPGRADE_TEST_IMAGE_NAME = System.getenv().getOrDefault("PULSAR_UPGRADE_TEST_IMAGE_NAME", From bbafb00a9a2eef798deee8ead70d3581bee12165 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Thu, 23 Jul 2026 17:20:44 +0800 Subject: [PATCH 131/213] [fix][broker] Check deliverAt before containsMessage in bucket addMessage (#26230) (cherry picked from commit 094f2702833ac0ddf71beae4df2072048861eaf5) --- .../bucket/BucketDelayedDeliveryTracker.java | 8 ++-- .../BucketDelayedDeliveryTrackerTest.java | 48 +++++++++++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index 6cf2876d4b880..37ee442b85a55 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -367,14 +367,14 @@ private void afterCreateImmutableBucket(Pair immu @Override public synchronized boolean addMessage(long ledgerId, long entryId, long deliverAt) { - if (containsMessage(ledgerId, entryId)) { - return true; - } - if (deliverAt < 0 || deliverAt <= getCutoffTime()) { return false; } + if (containsMessage(ledgerId, entryId)) { + return true; + } + boolean existBucket = findImmutableBucket(ledgerId).isPresent(); // Create bucket snapshot diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java index cfe7ed370daf9..4a73f335cf485 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java @@ -45,6 +45,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; +import lombok.Cleanup; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.Position; @@ -154,6 +155,10 @@ public Object[][] provider(Method method) throws Exception { new BucketDelayedDeliveryTracker(dispatcher, timer, 100000, clock, true, bucketSnapshotStorage, 20, TimeUnit.HOURS.toMillis(1), 5, 100) }}; + case "testExpiredTrackedMessageReturnsFalse", "testRecoverThenExpireAddMessage" -> new Object[][]{{ + new BucketDelayedDeliveryTracker(dispatcher, timer, 1, clock, + true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 50) + }}; default -> new Object[][]{{ new BucketDelayedDeliveryTracker(dispatcher, timer, 1, clock, true, bucketSnapshotStorage, 1000, TimeUnit.MILLISECONDS.toMillis(100), -1, 50) @@ -185,6 +190,49 @@ public void testContainsMessage(BucketDelayedDeliveryTracker tracker) { tracker.close(); } + @Test(dataProvider = "delayedTracker") + public void testExpiredTrackedMessageReturnsFalse(BucketDelayedDeliveryTracker tracker) { + clockTime.set(1000); + assertTrue(tracker.addMessage(1, 1, 2000)); + assertTrue(tracker.containsMessage(1, 1)); + + clockTime.set(2500); + + assertFalse( + "Expired tracked message should return false so dispatcher delivers immediately", + tracker.addMessage(1, 1, 2000)); + + tracker.close(); + } + + @Test(dataProvider = "delayedTracker") + public void testRecoverThenExpireAddMessage(BucketDelayedDeliveryTracker tracker) throws Exception { + clockTime.set(0); + for (int i = 1; i <= 6; i++) { + tracker.addMessage(i, i, i * 1000); + } + + Awaitility.await().untilAsserted(() -> + assertTrue(tracker.getImmutableBuckets().asMapOfRanges().values().stream() + .noneMatch(x -> x.merging || !x.getSnapshotCreateFuture().get().isDone()))); + + tracker.close(); + + clockTime.set(0); + @Cleanup + BucketDelayedDeliveryTracker tracker2 = new BucketDelayedDeliveryTracker( + dispatcher, timer, 100000, clock, true, + bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 50); + + assertTrue(tracker2.containsMessage(1, 1)); + + clockTime.set(10000); + + assertFalse( + "Recovered message that is now expired should return false", + tracker2.addMessage(1, 1, 1000)); + } + @Test(dataProvider = "delayedTracker", invocationCount = 10) public void testRecoverSnapshot(BucketDelayedDeliveryTracker tracker) throws Exception { for (int i = 1; i <= 100; i++) { From f2050555005c529f62af55d04f134985b3ce8505 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 23 Jul 2026 19:15:23 +0300 Subject: [PATCH 132/213] [fix][broker] Trigger max read position callback for messages published during transaction buffer recovery (#26234) (cherry picked from commit 520b0daa3f632e283930624bf29e7a5b4ed45828) --- .../buffer/impl/TopicTransactionBuffer.java | 21 ++- .../TopicTransactionBufferRecoveryTest.java | 144 ++++++++++++++++++ 2 files changed, 163 insertions(+), 2 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBufferRecoveryTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 77d28a6165407..997b455e12f28 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -150,7 +150,7 @@ private void recover() { public void recoverComplete() { synchronized (TopicTransactionBuffer.this) { if (ongoingTxns.isEmpty()) { - maxReadPosition = topic.getManagedLedger().getLastConfirmedEntry(); + updateMaxReadPositionAfterRecovery(); } if (!changeToReadyState()) { log.error("[{}]Transaction buffer recover fail, current state: {}", @@ -171,7 +171,7 @@ public void recoverComplete() { @Override public void noNeedToRecover() { synchronized (TopicTransactionBuffer.this) { - maxReadPosition = topic.getManagedLedger().getLastConfirmedEntry(); + updateMaxReadPositionAfterRecovery(); if (!changeToNoSnapshotState()) { log.error("[{}]Transaction buffer recover fail", topic.getName()); } else { @@ -624,6 +624,23 @@ void updateMaxReadPosition(Position newPosition, boolean disableCallback) { } } + /** + * Advance the max read position to the last confirmed entry when recovery finishes. While the transaction + * buffer is recovering, {@link #syncMaxReadPositionForNormalPublish(Position, boolean)} ignores publishes, so + * messages published during recovery would otherwise never trigger the maxReadPositionMovedForward callback. + * {@link PersistentTopic} uses that callback to maintain lastMaxReadPositionMovedForwardTimestamp, which + * ReplicatedSubscriptionsController relies on to detect new data when deciding whether to start a snapshot. + * Must be called while synchronized on this transaction buffer. + */ + private void updateMaxReadPositionAfterRecovery() { + Position preMaxReadPosition = this.maxReadPosition; + this.maxReadPosition = topic.getManagedLedger().getLastConfirmedEntry(); + if (this.maxReadPosition != null + && (preMaxReadPosition == null || preMaxReadPosition.compareTo(this.maxReadPosition) < 0)) { + this.maxReadPositionCallBack.maxReadPositionMovedForward(preMaxReadPosition, this.maxReadPosition); + } + } + @Override public CompletableFuture purgeTxns(List dataLedgers) { return null; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBufferRecoveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBufferRecoveryTest.java new file mode 100644 index 0000000000000..101f69b263f81 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBufferRecoveryTest.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.buffer.impl; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.CompletableFuture; +import lombok.Cleanup; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; +import org.apache.pulsar.broker.transaction.buffer.TransactionBufferProvider; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.awaitility.Awaitility; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +/** + * Tests for {@link TopicTransactionBuffer} recovery completion behavior. + */ +@Test(groups = "broker") +public class TopicTransactionBufferRecoveryTest extends ProducerConsumerBase { + + @BeforeClass(alwaysRun = true) + @Override + protected void setup() throws Exception { + conf.setTransactionCoordinatorEnabled(true); + super.internalSetup(); + super.producerBaseSetup(); + } + + @AfterClass(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @DataProvider(name = "snapshotExists") + public Object[][] snapshotExists() { + return new Object[][] { { false }, { true } }; + } + + /** + * While the transaction buffer is recovering, normal publishes don't move the max read position, so they + * don't update the topic's lastMaxReadPositionMovedForwardTimestamp either. When recovery completes, the + * transaction buffer must account for the messages published during recovery and trigger the + * maxReadPositionMovedForward callback; otherwise ReplicatedSubscriptionsController would consider the topic + * to have no new data and never start a subscription snapshot until further traffic arrives. + */ + @Test(dataProvider = "snapshotExists") + public void testMaxReadPositionMovedForwardForMessagesPublishedDuringRecovery(boolean snapshotExists) + throws Exception { + String tpName = BrokerTestUtil.newUniqueName("persistent://public/default/tp-tb-recovery"); + CompletableFuture recoverFuture = new CompletableFuture<>(); + TransactionBufferProvider originalProvider = pulsar.getTransactionBufferProvider(); + pulsar.setTransactionBufferProvider(originTopic -> { + AbortedTxnProcessor processor = mock(AbortedTxnProcessor.class); + when(processor.recoverFromSnapshot()).thenReturn(recoverFuture); + when(processor.closeAsync()).thenReturn(CompletableFuture.completedFuture(null)); + return new TopicTransactionBuffer( + (PersistentTopic) originTopic, processor, AbortedTxnProcessor.SnapshotType.Single); + }); + try { + @Cleanup + Producer producer = pulsarClient.newProducer().topic(tpName).create(); + // The transaction buffer is stuck in the recovering state until recoverFuture completes, so these + // messages are published while it is still recovering. + for (int i = 0; i < 3; i++) { + producer.send(("msg-" + i).getBytes(StandardCharsets.UTF_8)); + } + PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService() + .getTopicIfExists(tpName).get().orElseThrow(); + assertEquals(persistentTopic.getLastMaxReadPositionMovedForwardTimestamp(), 0L, + "Publishes during recovery are not expected to move the max read position forward"); + + // Let the recovery finish, either replaying from a snapshot position or without any snapshot. + recoverFuture.complete(snapshotExists ? PositionFactory.EARLIEST : null); + + Awaitility.await().untilAsserted(() -> { + assertTrue(persistentTopic.getLastMaxReadPositionMovedForwardTimestamp() > 0, + "Completed recovery should move the max read position forward for the messages" + + " published during recovery"); + assertEquals(persistentTopic.getTransactionBuffer().getMaxReadPosition(), + persistentTopic.getManagedLedger().getLastConfirmedEntry()); + }); + } finally { + pulsar.setTransactionBufferProvider(originalProvider); + } + } + + /** + * When nothing is published while the transaction buffer recovers, completing the recovery must not trigger + * the maxReadPositionMovedForward callback, so an idle topic isn't mistaken for one with new data. + */ + @Test + public void testMaxReadPositionNotMovedForwardWhenNothingPublishedDuringRecovery() throws Exception { + String tpName = BrokerTestUtil.newUniqueName("persistent://public/default/tp-tb-recovery-idle"); + CompletableFuture recoverFuture = new CompletableFuture<>(); + TransactionBufferProvider originalProvider = pulsar.getTransactionBufferProvider(); + pulsar.setTransactionBufferProvider(originTopic -> { + AbortedTxnProcessor processor = mock(AbortedTxnProcessor.class); + when(processor.recoverFromSnapshot()).thenReturn(recoverFuture); + when(processor.closeAsync()).thenReturn(CompletableFuture.completedFuture(null)); + return new TopicTransactionBuffer( + (PersistentTopic) originTopic, processor, AbortedTxnProcessor.SnapshotType.Single); + }); + try { + PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService() + .getTopic(tpName, true).get().orElseThrow(); + recoverFuture.complete(null); + Awaitility.await().untilAsserted(() -> + assertEquals(persistentTopic.getTransactionBuffer().getMaxReadPosition(), + persistentTopic.getManagedLedger().getLastConfirmedEntry())); + assertEquals(persistentTopic.getLastMaxReadPositionMovedForwardTimestamp(), 0L, + "Recovery of an idle topic should not move the max read position forward"); + } finally { + pulsar.setTransactionBufferProvider(originalProvider); + } + } +} From dbd8ca0c32fce935ef30cc2e1aeaa838252698f7 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 23 Jul 2026 22:15:14 +0300 Subject: [PATCH 133/213] [improve][misc] Upgrade Jetty to 12.1.11 (#26233) (cherry picked from commit 4bfb08063e753fa3d9b982b283f8cefe94dc6230) --- .../server/src/assemble/LICENSE.bin.txt | 72 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 24 +++---- pom.xml | 2 +- 3 files changed, 49 insertions(+), 49 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index befe6ffc7aa16..a0bd04d1d6aa0 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -393,43 +393,43 @@ The Apache Software License, Version 2.0 - org.asynchttpclient-async-http-client-2.15.0.jar - org.asynchttpclient-async-http-client-netty-utils-2.15.0.jar * Jetty - - org.eclipse.jetty-jetty-alpn-client-12.1.10.jar - - org.eclipse.jetty-jetty-alpn-conscrypt-server-12.1.10.jar - - org.eclipse.jetty-jetty-alpn-server-12.1.10.jar - - org.eclipse.jetty-jetty-annotations-12.1.10.jar - - org.eclipse.jetty-jetty-client-12.1.10.jar - - org.eclipse.jetty-jetty-http-12.1.10.jar - - org.eclipse.jetty-jetty-io-12.1.10.jar - - org.eclipse.jetty-jetty-jndi-12.1.10.jar - - org.eclipse.jetty-jetty-plus-12.1.10.jar - - org.eclipse.jetty-jetty-security-12.1.10.jar - - org.eclipse.jetty-jetty-server-12.1.10.jar - - org.eclipse.jetty-jetty-session-12.1.10.jar - - org.eclipse.jetty-jetty-util-12.1.10.jar - - org.eclipse.jetty-jetty-xml-12.1.10.jar - - org.eclipse.jetty.compression-jetty-compression-common-12.1.10.jar - - org.eclipse.jetty.compression-jetty-compression-gzip-12.1.10.jar - - org.eclipse.jetty.compression-jetty-compression-server-12.1.10.jar - - org.eclipse.jetty.ee-jetty-ee-webapp-12.1.10.jar - - org.eclipse.jetty.ee8-jetty-ee8-annotations-12.1.10.jar - - org.eclipse.jetty.ee8-jetty-ee8-nested-12.1.10.jar - - org.eclipse.jetty.ee8-jetty-ee8-plus-12.1.10.jar - - org.eclipse.jetty.ee8-jetty-ee8-proxy-12.1.10.jar - - org.eclipse.jetty.ee8-jetty-ee8-security-12.1.10.jar - - org.eclipse.jetty.ee8-jetty-ee8-servlet-12.1.10.jar - - org.eclipse.jetty.ee8-jetty-ee8-servlets-12.1.10.jar - - org.eclipse.jetty.ee8-jetty-ee8-webapp-12.1.10.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-api-12.1.10.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-common-12.1.10.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-server-12.1.10.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-servlet-12.1.10.jar + - org.eclipse.jetty-jetty-alpn-client-12.1.11.jar + - org.eclipse.jetty-jetty-alpn-conscrypt-server-12.1.11.jar + - org.eclipse.jetty-jetty-alpn-server-12.1.11.jar + - org.eclipse.jetty-jetty-annotations-12.1.11.jar + - org.eclipse.jetty-jetty-client-12.1.11.jar + - org.eclipse.jetty-jetty-http-12.1.11.jar + - org.eclipse.jetty-jetty-io-12.1.11.jar + - org.eclipse.jetty-jetty-jndi-12.1.11.jar + - org.eclipse.jetty-jetty-plus-12.1.11.jar + - org.eclipse.jetty-jetty-security-12.1.11.jar + - org.eclipse.jetty-jetty-server-12.1.11.jar + - org.eclipse.jetty-jetty-session-12.1.11.jar + - org.eclipse.jetty-jetty-util-12.1.11.jar + - org.eclipse.jetty-jetty-xml-12.1.11.jar + - org.eclipse.jetty.compression-jetty-compression-common-12.1.11.jar + - org.eclipse.jetty.compression-jetty-compression-gzip-12.1.11.jar + - org.eclipse.jetty.compression-jetty-compression-server-12.1.11.jar + - org.eclipse.jetty.ee-jetty-ee-webapp-12.1.11.jar + - org.eclipse.jetty.ee8-jetty-ee8-annotations-12.1.11.jar + - org.eclipse.jetty.ee8-jetty-ee8-nested-12.1.11.jar + - org.eclipse.jetty.ee8-jetty-ee8-plus-12.1.11.jar + - org.eclipse.jetty.ee8-jetty-ee8-proxy-12.1.11.jar + - org.eclipse.jetty.ee8-jetty-ee8-security-12.1.11.jar + - org.eclipse.jetty.ee8-jetty-ee8-servlet-12.1.11.jar + - org.eclipse.jetty.ee8-jetty-ee8-servlets-12.1.11.jar + - org.eclipse.jetty.ee8-jetty-ee8-webapp-12.1.11.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-api-12.1.11.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-common-12.1.11.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-server-12.1.11.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-servlet-12.1.11.jar - org.eclipse.jetty.toolchain-jetty-servlet-api-4.0.9.jar - - org.eclipse.jetty.websocket-jetty-websocket-core-client-12.1.10.jar - - org.eclipse.jetty.websocket-jetty-websocket-core-common-12.1.10.jar - - org.eclipse.jetty.websocket-jetty-websocket-core-server-12.1.10.jar - - org.eclipse.jetty.websocket-jetty-websocket-jetty-api-12.1.10.jar - - org.eclipse.jetty.websocket-jetty-websocket-jetty-client-12.1.10.jar - - org.eclipse.jetty.websocket-jetty-websocket-jetty-common-12.1.10.jar + - org.eclipse.jetty.websocket-jetty-websocket-core-client-12.1.11.jar + - org.eclipse.jetty.websocket-jetty-websocket-core-common-12.1.11.jar + - org.eclipse.jetty.websocket-jetty-websocket-core-server-12.1.11.jar + - org.eclipse.jetty.websocket-jetty-websocket-jetty-api-12.1.11.jar + - org.eclipse.jetty.websocket-jetty-websocket-jetty-client-12.1.11.jar + - org.eclipse.jetty.websocket-jetty-websocket-jetty-common-12.1.11.jar * SnakeYaml -- org.yaml-snakeyaml-2.0.jar * RocksDB - org.rocksdb-rocksdbjni-7.9.2.jar * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.45.0.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index f664b0971d6d3..21ea126a65e3d 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -401,18 +401,18 @@ The Apache Software License, Version 2.0 - async-http-client-2.15.0.jar - async-http-client-netty-utils-2.15.0.jar * Jetty - - jetty-alpn-client-12.1.10.jar - - jetty-client-12.1.10.jar - - jetty-compression-common-12.1.10.jar - - jetty-compression-gzip-12.1.10.jar - - jetty-http-12.1.10.jar - - jetty-io-12.1.10.jar - - jetty-util-12.1.10.jar - - jetty-websocket-core-client-12.1.10.jar - - jetty-websocket-core-common-12.1.10.jar - - jetty-websocket-jetty-api-12.1.10.jar - - jetty-websocket-jetty-client-12.1.10.jar - - jetty-websocket-jetty-common-12.1.10.jar + - jetty-alpn-client-12.1.11.jar + - jetty-client-12.1.11.jar + - jetty-compression-common-12.1.11.jar + - jetty-compression-gzip-12.1.11.jar + - jetty-http-12.1.11.jar + - jetty-io-12.1.11.jar + - jetty-util-12.1.11.jar + - jetty-websocket-core-client-12.1.11.jar + - jetty-websocket-core-common-12.1.11.jar + - jetty-websocket-jetty-api-12.1.11.jar + - jetty-websocket-jetty-client-12.1.11.jar + - jetty-websocket-jetty-common-12.1.11.jar * SnakeYaml -- snakeyaml-2.0.jar * Google Error Prone Annotations - error_prone_annotations-2.45.0.jar * Javassist -- javassist-3.25.0-GA.jar diff --git a/pom.xml b/pom.xml index 7ce562c2ad039..a2cb40d28d0e8 100644 --- a/pom.xml +++ b/pom.xml @@ -189,7 +189,7 @@ flexible messaging model and an intuitive client API. 5.7.1 4.1.136.Final 0.0.26.Final - 12.1.10 + 12.1.11 9.4.58.v20250814 2.5.2 From e962fa3438172f263fac541f007cfd8711d3f21c Mon Sep 17 00:00:00 2001 From: kannar Date: Sat, 25 Jul 2026 10:37:14 +0200 Subject: [PATCH 134/213] [improve][offload] Support credentials from offload policies for S3 and Aliyun OSS drivers (#26232) Co-authored-by: Claude Fable 5 (cherry picked from commit a965f5f7df3e060d9a9d4e63caf0561a2b90441c) --- .../policies/data/OffloadPoliciesImpl.java | 26 +++++++---- .../policies/data/OffloadPoliciesTest.java | 46 +++++++++++++++++++ .../provider/JCloudBlobStoreProvider.java | 19 ++++++++ .../provider/JCloudBlobStoreProviderTest.java | 46 +++++++++++++++++++ 4 files changed, 128 insertions(+), 9 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java index 8c8432644adbf..14b08e05caf3c 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java @@ -64,9 +64,15 @@ public class OffloadPoliciesImpl implements Serializable, OffloadPolicies { CONFIGURATION_FIELDS = Collections.unmodifiableList(temp); } - public static final ImmutableList INTERNAL_SUPPORTED_DRIVER = ImmutableList.of("S3", - "aws-s3", "google-cloud-storage", "filesystem", "azureblob", "aliyun-oss"); - public static final ImmutableList DRIVER_NAMES; + public static final String DRIVER_S3 = "S3"; + public static final String DRIVER_AWS_S3 = "aws-s3"; + public static final String DRIVER_GOOGLE_CLOUD_STORAGE = "google-cloud-storage"; + public static final String DRIVER_FILESYSTEM = "filesystem"; + public static final String DRIVER_AZUREBLOB = "azureblob"; + public static final String DRIVER_ALIYUN_OSS = "aliyun-oss"; + public static final List INTERNAL_SUPPORTED_DRIVER = Arrays.asList(DRIVER_S3, + DRIVER_AWS_S3, DRIVER_GOOGLE_CLOUD_STORAGE, DRIVER_FILESYSTEM, DRIVER_AZUREBLOB, DRIVER_ALIYUN_OSS); + public static final List DRIVER_NAMES; static { String extraDrivers = System.getProperty("pulsar.extra.offload.drivers", ""); if (extraDrivers.trim().isEmpty()) { @@ -222,7 +228,8 @@ public static OffloadPoliciesImpl create(String driver, String region, String bu .managedLedgerOffloadReadBufferSizeInBytes(readBufferSizeInBytes) .managedLedgerOffloadedReadPriority(readPriority); - if (driver.equalsIgnoreCase(DRIVER_NAMES.get(0)) || driver.equalsIgnoreCase(DRIVER_NAMES.get(1))) { + if (driver.equalsIgnoreCase(DRIVER_S3) || driver.equalsIgnoreCase(DRIVER_AWS_S3) + || driver.equalsIgnoreCase(DRIVER_ALIYUN_OSS)) { if (role != null) { builder.s3ManagedLedgerOffloadRole(role); } @@ -241,7 +248,7 @@ public static OffloadPoliciesImpl create(String driver, String region, String bu .s3ManagedLedgerOffloadServiceEndpoint(endpoint) .s3ManagedLedgerOffloadMaxBlockSizeInBytes(maxBlockSizeInBytes) .s3ManagedLedgerOffloadReadBufferSizeInBytes(readBufferSizeInBytes); - } else if (driver.equalsIgnoreCase(DRIVER_NAMES.get(2))) { + } else if (driver.equalsIgnoreCase(DRIVER_GOOGLE_CLOUD_STORAGE)) { builder.gcsManagedLedgerOffloadRegion(region) .gcsManagedLedgerOffloadBucket(bucket) .gcsManagedLedgerOffloadMaxBlockSizeInBytes(maxBlockSizeInBytes) @@ -311,22 +318,23 @@ public boolean isS3Driver() { if (managedLedgerOffloadDriver == null) { return false; } - return managedLedgerOffloadDriver.equalsIgnoreCase(DRIVER_NAMES.get(0)) - || managedLedgerOffloadDriver.equalsIgnoreCase(DRIVER_NAMES.get(1)); + return managedLedgerOffloadDriver.equalsIgnoreCase(DRIVER_S3) + || managedLedgerOffloadDriver.equalsIgnoreCase(DRIVER_AWS_S3) + || managedLedgerOffloadDriver.equalsIgnoreCase(DRIVER_ALIYUN_OSS); } public boolean isGcsDriver() { if (managedLedgerOffloadDriver == null) { return false; } - return managedLedgerOffloadDriver.equalsIgnoreCase(DRIVER_NAMES.get(2)); + return managedLedgerOffloadDriver.equalsIgnoreCase(DRIVER_GOOGLE_CLOUD_STORAGE); } public boolean isFileSystemDriver() { if (managedLedgerOffloadDriver == null) { return false; } - return managedLedgerOffloadDriver.equalsIgnoreCase(DRIVER_NAMES.get(3)); + return managedLedgerOffloadDriver.equalsIgnoreCase(DRIVER_FILESYSTEM); } public boolean bucketValid() { diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/OffloadPoliciesTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/OffloadPoliciesTest.java index 62934343e59e8..76912b9563ad7 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/OffloadPoliciesTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/OffloadPoliciesTest.java @@ -97,6 +97,52 @@ public void testS3Configuration() { Assert.assertEquals(offloadPolicies.getManagedLedgerOffloadThresholdInSeconds(), offloadThresholdInSeconds); } + @Test + public void testAliyunOssConfiguration() { + final String driver = "aliyun-oss"; + final String region = "test-region"; + final String bucket = "test-bucket"; + final String role = "test-role"; + final String roleSessionName = "test-role-session-name"; + final String credentialId = "test-credential-id"; + final String credentialSecret = "test-credential-secret"; + final String endPoint = "test-endpoint"; + final Integer maxBlockSizeInBytes = 5 * M; + final Integer readBufferSizeInBytes = 2 * M; + final Long offloadThresholdInBytes = 10L * M; + final Long offloadThresholdInSeconds = 1000L; + final Long offloadDeletionLagInMillis = 5L * MIN; + + OffloadPoliciesImpl offloadPolicies = OffloadPoliciesImpl.create( + driver, + region, + bucket, + endPoint, + role, + roleSessionName, + credentialId, + credentialSecret, + maxBlockSizeInBytes, + readBufferSizeInBytes, + offloadThresholdInBytes, + offloadThresholdInSeconds, + offloadDeletionLagInMillis, + OffloadedReadPriority.TIERED_STORAGE_FIRST + ); + + Assert.assertTrue(offloadPolicies.isS3Driver()); + Assert.assertEquals(offloadPolicies.getManagedLedgerOffloadDriver(), driver); + Assert.assertEquals(offloadPolicies.getS3ManagedLedgerOffloadRegion(), region); + Assert.assertEquals(offloadPolicies.getS3ManagedLedgerOffloadBucket(), bucket); + Assert.assertEquals(offloadPolicies.getS3ManagedLedgerOffloadServiceEndpoint(), endPoint); + // The CLI create() path must carry the credentials under the s3-prefixed keys the + // S3-compatible aliyun-oss offloader reads, not silently drop them. + Assert.assertEquals(offloadPolicies.getS3ManagedLedgerOffloadCredentialId(), credentialId); + Assert.assertEquals(offloadPolicies.getS3ManagedLedgerOffloadCredentialSecret(), credentialSecret); + Assert.assertEquals(offloadPolicies.getS3ManagedLedgerOffloadMaxBlockSizeInBytes(), maxBlockSizeInBytes); + Assert.assertEquals(offloadPolicies.getS3ManagedLedgerOffloadReadBufferSizeInBytes(), readBufferSizeInBytes); + } + @Test public void testGcsConfiguration() { final String driver = "google-cloud-storage"; diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/JCloudBlobStoreProvider.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/JCloudBlobStoreProvider.java index f859f2b73284b..b8d30935b8d8a 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/JCloudBlobStoreProvider.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/JCloudBlobStoreProvider.java @@ -429,6 +429,25 @@ public String getAWSSecretKey() { }; static final CredentialBuilder S3_CREDENTIAL_BUILDER = (TieredStorageConfiguration config) -> { + if (config.getCredentials() != null) { + return; + } + // Credentials provided in the tiered storage configuration take + // precedence over the environment variables. Offload policies carry + // credentials under the s3-prefixed keys for every S3-compatible + // driver (see OffloadPoliciesImpl), so those are the keys accepted. + String configId = config.getConfigProperty(S3_ID_FIELD); + String configSecret = config.getConfigProperty(S3_SECRET_FIELD); + if (StringUtils.isNotBlank(configId) || StringUtils.isNotBlank(configSecret)) { + if (StringUtils.isBlank(configId) || StringUtils.isBlank(configSecret)) { + throw new IllegalArgumentException( + "Both " + S3_ID_FIELD + " and " + S3_SECRET_FIELD + + " must be set when providing offload credentials in the configuration"); + } + Credentials credentials = new Credentials(configId, configSecret); + config.setProviderCredentials(() -> credentials); + return; + } String accountName = System.getenv().getOrDefault("ACCESS_KEY_ID", ""); // For forward compatibility if (StringUtils.isEmpty(accountName.trim())) { diff --git a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/JCloudBlobStoreProviderTest.java b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/JCloudBlobStoreProviderTest.java index f45cb30ea4c2b..7ad2eef99474f 100644 --- a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/JCloudBlobStoreProviderTest.java +++ b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/JCloudBlobStoreProviderTest.java @@ -19,8 +19,10 @@ package org.apache.bookkeeper.mledger.offload.jcloud.provider; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; import java.util.HashMap; import java.util.Map; +import org.jclouds.domain.Credentials; import org.testng.annotations.Test; public class JCloudBlobStoreProviderTest { @@ -130,4 +132,48 @@ public void s3ValidationBucketMissed() { TieredStorageConfiguration configuration = new TieredStorageConfiguration(map); configuration.getProvider().validate(configuration); } + + // Offload policies (OffloadPoliciesImpl) carry credentials under the + // s3-prefixed keys for every S3-compatible driver + private TieredStorageConfiguration credentialConfig(String driver, String id, String secret) { + Map map = new HashMap<>(); + map.put("managedLedgerOffloadDriver", driver); + map.put("managedLedgerOffloadServiceEndpoint", "http://storage.service"); + map.put("managedLedgerOffloadBucket", "test-bucket"); + if (id != null) { + map.put("s3ManagedLedgerOffloadCredentialId", id); + } + if (secret != null) { + map.put("s3ManagedLedgerOffloadCredentialSecret", secret); + } + return new TieredStorageConfiguration(map); + } + + private void assertCredentialsFromConfig(String driver) { + TieredStorageConfiguration configuration = + credentialConfig(driver, "config-access-id", "config-access-secret"); + configuration.getProvider().buildCredentials(configuration); + assertNotNull(configuration.getProviderCredentials()); + Credentials credentials = configuration.getProviderCredentials().get(); + assertEquals(credentials.identity, "config-access-id"); + assertEquals(credentials.credential, "config-access-secret"); + } + + @Test + public void s3CredentialsFromConfigTest() { + assertCredentialsFromConfig("S3"); + } + + @Test + public void aliyunOssCredentialsFromConfigTest() { + assertCredentialsFromConfig("aliyun-oss"); + } + + @Test(expectedExceptions = IllegalArgumentException.class, + expectedExceptionsMessageRegExp = "Both s3ManagedLedgerOffloadCredentialId and " + + "s3ManagedLedgerOffloadCredentialSecret must be set.*") + public void s3PartialCredentialsFromConfigTest() { + TieredStorageConfiguration configuration = credentialConfig("S3", "config-access-id", null); + configuration.getProvider().buildCredentials(configuration); + } } From ed2f0b10eeef19a639dcaa77349d9b91727486eb Mon Sep 17 00:00:00 2001 From: SongOf <46475785+SongOf@users.noreply.github.com> Date: Sat, 25 Jul 2026 16:38:35 +0800 Subject: [PATCH 135/213] [fix][meta] Fix RocksdbMetadataStore instanceId not advancing across restarts (#26218) Co-authored-by: maxlisongsong (cherry picked from commit a4b7aaa616959af35e6bcd381a24fdf076acd3ac) --- .../org/apache/pulsar/metadata/impl/RocksdbMetadataStore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/RocksdbMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/RocksdbMetadataStore.java index 08e5478ffcca1..35b914e665b0f 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/RocksdbMetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/RocksdbMetadataStore.java @@ -264,7 +264,7 @@ private AtomicLong loadSequentialIdGenerator() throws RocksDBException { if (value != null) { generator.set(toLong(value)); } else { - db.put(writeOptions, INSTANCE_ID_KEY, toBytes(generator.get())); + db.put(writeOptions, SEQUENTIAL_ID_KEY, toBytes(generator.get())); } return generator; } From 52f6c51616ddd9d25fe3acb376b474af140b5543 Mon Sep 17 00:00:00 2001 From: SongOf <46475785+SongOf@users.noreply.github.com> Date: Sat, 25 Jul 2026 19:15:45 +0800 Subject: [PATCH 136/213] [fix][meta] Record get op stats on the correct completion branch in AbstractMetadataStore (#26201) Co-authored-by: maxlisongsong (cherry picked from commit 78cc3f41c510b17d5b2569811e62c5744280401e) --- .../metadata/impl/AbstractMetadataStore.java | 2 +- .../AbstractMetadataStoreGetStatsTest.java | 122 ++++++++++++++++++ 2 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/AbstractMetadataStoreGetStatsTest.java diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java index 92b1b9e2b7f80..53ccf09e297d8 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java @@ -301,9 +301,9 @@ public CompletableFuture> get(String path) { return storeGet(path) .whenComplete((v, t) -> { if (t != null) { - v.ifPresent(getResult -> nodeSizeStats.recordGetRes(path, getResult)); metadataStoreStats.recordGetOpsFailed(System.currentTimeMillis() - start); } else { + v.ifPresent(getResult -> nodeSizeStats.recordGetRes(path, getResult)); metadataStoreStats.recordGetOpsSucceeded(System.currentTimeMillis() - start); } }); diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/AbstractMetadataStoreGetStatsTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/AbstractMetadataStoreGetStatsTest.java new file mode 100644 index 0000000000000..a90bf4353e724 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/AbstractMetadataStoreGetStatsTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.metadata.impl; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertSame; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; +import io.opentelemetry.api.OpenTelemetry; +import io.prometheus.client.CollectorRegistry; +import java.util.Arrays; +import java.util.EnumSet; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.function.Function; +import lombok.Cleanup; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.metadata.api.GetResult; +import org.apache.pulsar.metadata.api.MetadataNodeSizeStats; +import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.apache.pulsar.metadata.api.Stat; +import org.apache.pulsar.metadata.api.extended.CreateOption; +import org.testng.annotations.Test; + +/** + * Verifies that {@link AbstractMetadataStore#get(String)} records operation stats on the + * correct completion branch. + */ +public class AbstractMetadataStoreGetStatsTest { + + private static class TestMetadataStore extends AbstractMetadataStore { + private volatile Function>> getImpl; + + TestMetadataStore(String name, MetadataNodeSizeStats nodeSizeStats) { + super(name, OpenTelemetry.noop(), nodeSizeStats, 1); + } + + @Override + protected CompletableFuture> storeGet(String path) { + return getImpl.apply(path); + } + + @Override + public CompletableFuture> getChildrenFromStore(String path) { + return CompletableFuture.completedFuture(List.of()); + } + + @Override + protected CompletableFuture existsFromStore(String path) { + return CompletableFuture.completedFuture(false); + } + + @Override + protected CompletableFuture storeDelete(String path, Optional expectedVersion) { + return CompletableFuture.completedFuture(null); + } + + @Override + protected CompletableFuture storePut(String path, byte[] data, Optional optExpectedVersion, + EnumSet options) { + return FutureUtil.failedFuture(new UnsupportedOperationException()); + } + } + + private static double getOpsCount(String storeName, String status) { + Double value = CollectorRegistry.defaultRegistry.getSampleValue( + "pulsar_metadata_store_ops_latency_ms_count", + new String[]{"name", "type", "status"}, + new String[]{storeName, "get", status}); + return value == null ? 0.0 : value; + } + + @Test + public void testGetSuccessRecordsNodeSizeStats() throws Exception { + MetadataNodeSizeStats nodeSizeStats = mock(MetadataNodeSizeStats.class); + @Cleanup + TestMetadataStore store = new TestMetadataStore("get-stats-success", nodeSizeStats); + GetResult result = new GetResult(new byte[]{1, 2, 3}, new Stat("/a", 0, 0, 0, false, false)); + store.getImpl = path -> CompletableFuture.completedFuture(Optional.of(result)); + + assertSame(store.get("/a").join().orElse(null), result); + + verify(nodeSizeStats).recordGetRes("/a", result); + assertEquals(getOpsCount("get-stats-success", "success"), 1.0); + } + + @Test + public void testGetFailureRecordsFailedOpAndPropagatesOriginalException() throws Exception { + String storeName = "get-stats-failure"; + @Cleanup + TestMetadataStore store = new TestMetadataStore(storeName, mock(MetadataNodeSizeStats.class)); + MetadataStoreException injected = new MetadataStoreException("injected failure"); + store.getImpl = path -> FutureUtil.failedFuture(injected); + + CompletionException ex = expectThrows(CompletionException.class, () -> store.get("/a").join()); + + assertSame(ex.getCause(), injected); + assertTrue(Arrays.stream(injected.getSuppressed()).noneMatch(s -> s instanceof NullPointerException), + "stats callback must not throw NPE on the failure path"); + assertEquals(getOpsCount(storeName, "fail"), 1.0); + } +} From ba9d4f2b3b4cb225339c84bf7840bc8c7a43f5be Mon Sep 17 00:00:00 2001 From: SongOf <46475785+SongOf@users.noreply.github.com> Date: Sat, 25 Jul 2026 20:40:47 +0800 Subject: [PATCH 137/213] [fix][meta] Complete handleMetadataEvent future exceptionally when the initial get fails (#26199) Co-authored-by: maxlisongsong (cherry picked from commit bc56ba8d30b2a96648a041359579b1e1138fb6f3) --- .../metadata/impl/AbstractMetadataStore.java | 10 ++++-- .../impl/MetadataEventSynchronizerTest.java | 31 +++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java index 53ccf09e297d8..3cd680514df3a 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java @@ -167,14 +167,14 @@ public CompletableFuture asyncReload(String key, Boolean oldValue, @Override public CompletableFuture handleMetadataEvent(MetadataEvent event) { CompletableFuture result = new CompletableFuture<>(); - get(event.getPath()).thenApply(res -> { + get(event.getPath()).thenAccept(res -> { Set options = event.getOptions() != null ? event.getOptions() : Collections.emptySet(); if (res.isPresent()) { GetResult existingValue = res.get(); if (shouldIgnoreEvent(event, existingValue)) { result.complete(null); - return result; + return; } } // else update the event @@ -196,7 +196,11 @@ public CompletableFuture handleMetadataEvent(MetadataEvent event) { } return false; }); - return result; + }).exceptionally(ex -> { + Throwable cause = FutureUtil.unwrapCompletionException(ex); + log.warn("Failed to handle metadata event {}", event.getPath(), cause); + result.completeExceptionally(cause); + return null; }); return result; } diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/MetadataEventSynchronizerTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/MetadataEventSynchronizerTest.java index 3b07e0b3c2bd6..90c17b7c5ae44 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/MetadataEventSynchronizerTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/MetadataEventSynchronizerTest.java @@ -20,13 +20,22 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; import java.nio.charset.StandardCharsets; +import java.util.HashSet; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import lombok.Cleanup; +import org.apache.pulsar.metadata.api.GetResult; +import org.apache.pulsar.metadata.api.MetadataEvent; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.MetadataStoreConfig; +import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.MetadataStoreFactory; +import org.apache.pulsar.metadata.api.NotificationType; import org.awaitility.Awaitility; import org.testng.annotations.Test; @@ -75,6 +84,28 @@ public void testSharedInstance() throws Exception { }); } + @Test + public void testHandleMetadataEventCompletesWhenGetFails() throws Exception { + @Cleanup + LocalMemoryMetadataStore store = new LocalMemoryMetadataStore("memory:local", + MetadataStoreConfig.builder().build()) { + @Override + public CompletableFuture> storeGet(String path) { + return CompletableFuture.failedFuture( + new MetadataStoreException("injected storeGet failure")); + } + }; + + MetadataEvent event = new MetadataEvent("/test", "value".getBytes(StandardCharsets.UTF_8), + new HashSet<>(), null, System.currentTimeMillis(), "test-cluster", NotificationType.Modified); + + CompletableFuture result = store.handleMetadataEvent(event); + // The future must not hang when the initial get() fails: it should complete exceptionally + ExecutionException ex = expectThrows(ExecutionException.class, () -> result.get(5, TimeUnit.SECONDS)); + assertTrue(ex.getCause() instanceof MetadataStoreException, + "expected MetadataStoreException cause but got: " + ex.getCause()); + } + @Test public void testPathValid() { assertFalse(AbstractMetadataStore.isValidPath(null)); From 80b6f4bbfc8a368efa8883266a12c86f7117a46a Mon Sep 17 00:00:00 2001 From: Alexandre Burgoni <23573685+alexandrebrg@users.noreply.github.com> Date: Fri, 24 Jul 2026 00:02:34 +0200 Subject: [PATCH 138/213] [fix][broker] Prevent stale read completions from stranding Failover subscriptions (#26174) Signed-off-by: Alexandre Burgoni (cherry picked from commit 0679b0d897de064bd6edfd47694f35a9ff44e80f) (cherry picked from commit 8cbbb6e6e59f5c37756781024e0db051b8889104) --- ...sistentDispatcherSingleActiveConsumer.java | 39 +- .../broker/service/PersistentTopicTest.java | 2 +- ...cherSingleActiveConsumerStuckReadTest.java | 671 ++++++++++++++++++ ...entDispatcherSingleActiveConsumerTest.java | 2 +- 4 files changed, 708 insertions(+), 6 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumerStuckReadTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java index 0e568aa41c492..e48db73db4f31 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java @@ -71,6 +71,11 @@ public class PersistentDispatcherSingleActiveConsumer extends AbstractDispatcher private Optional dispatchRateLimiter = Optional.empty(); protected volatile boolean havePendingRead = false; + // Monotonic identity of the read that havePendingRead currently refers to. Bumped under this monitor + // whenever readMoreEntries issues a read; a completion mutates dispatcher state only when its captured + // epoch still matches, so a completion whose read was disowned by a redeliver-driven rewind + re-arm + // is ignored instead of clearing havePendingRead for the newer outstanding read. Guarded by "this". + private long readOpEpoch = 0L; protected volatile int readBatchSize; protected final Backoff readFailureBackoff; @@ -153,7 +158,21 @@ protected void cancelPendingRead() { } } - private synchronized void readEntriesComplete(List entries, Consumer readConsumer, long epoch) { + private synchronized void readEntriesComplete(List entries, Consumer readConsumer, long epoch, + long readOpEpoch) { + if (readOpEpoch != this.readOpEpoch) { + // Stale completion: this read was disowned by a redeliver-driven cursor rewind + re-arm + // (internalRedeliverUnacknowledgedMessages) that already issued a newer read. Clearing + // havePendingRead or dispatching here would strand the newer armed read and double-deliver; + // release the entries (the newer read re-delivers from the rewound position) and do nothing else. + if (log.isDebugEnabled()) { + log.debug("[{}-{}] Discarding stale read completion: size={}, staleReadOpEpoch={}," + + " currentReadOpEpoch={}", + name, readConsumer, entries.size(), readOpEpoch, this.readOpEpoch); + } + entries.forEach(Entry::release); + return; + } if (log.isDebugEnabled()) { log.debug("[{}-{}] Got messages: {}", name, readConsumer, entries.size()); } @@ -366,6 +385,9 @@ void readMoreEntries(Consumer consumer) { log.debug("[{}-{}] Schedule read of {} messages", name, consumer, messagesToRead); } havePendingRead = true; + // Tag this read so a completion that has since been superseded by a redeliver-driven re-arm + // can be detected as stale (see readEntriesComplete / readEntriesFailed). + final long readOpEpoch = ++this.readOpEpoch; // TODO: should we pass the consumer epoch for compacted read path? See // https://github.com/apache/pulsar/issues/13690 final var epoch = consumer.readCompacted() ? DEFAULT_CONSUMER_EPOCH : consumer.getConsumerEpoch(); @@ -382,9 +404,9 @@ void readMoreEntries(Consumer consumer) { } entriesFuture.whenCompleteAsync((entries, e) -> { if (e == null) { - readEntriesComplete(entries, consumer, epoch); + readEntriesComplete(entries, consumer, epoch, readOpEpoch); } else { - readEntriesFailed(e, consumer); + readEntriesFailed(e, consumer, readOpEpoch); } }, executor); } @@ -457,7 +479,16 @@ protected Pair calculateToRead(Consumer consumer) { } @VisibleForTesting - public synchronized void readEntriesFailed(Throwable throwable, Consumer consumer) { + public synchronized void readEntriesFailed(Throwable throwable, Consumer consumer, long readOpEpoch) { + if (readOpEpoch != this.readOpEpoch) { + // Stale failure for a read already disowned by a redeliver-driven re-arm; clearing havePendingRead + // or rescheduling here would disturb the newer outstanding read. Ignore it. + if (log.isDebugEnabled()) { + log.debug("[{}-{}] Ignoring stale read failure: staleReadOpEpoch={}, currentReadOpEpoch={}", + name, consumer, readOpEpoch, this.readOpEpoch); + } + return; + } havePendingRead = false; final var exception = FutureUtil.unwrapCompletionException(throwable); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java index f5ec69e43e544..7b1ec8b8ac09d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java @@ -380,7 +380,7 @@ public void testDispatcherSingleConsumerReadFailed() { PersistentDispatcherSingleActiveConsumer dispatcher = new PersistentDispatcherSingleActiveConsumer(cursor, SubType.Exclusive, 1, topic, null); Consumer consumer = mock(Consumer.class); - dispatcher.readEntriesFailed(new ManagedLedgerException.InvalidCursorPositionException("failed"), consumer); + dispatcher.readEntriesFailed(new ManagedLedgerException.InvalidCursorPositionException("failed"), consumer, 0L); verify(topic, atLeast(1)).getBrokerService(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumerStuckReadTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumerStuckReadTest.java new file mode 100644 index 0000000000000..9d2c54a4f34a8 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumerStuckReadTest.java @@ -0,0 +1,671 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service.persistent; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import io.netty.util.concurrent.ImmediateEventExecutor; +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.AbstractExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.common.util.OrderedExecutor; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.ManagedLedgerConfig; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.bookkeeper.test.MockedBookKeeperTestCase; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.service.BrokerService; +import org.apache.pulsar.broker.service.Consumer; +import org.apache.pulsar.broker.service.Dispatcher; +import org.apache.pulsar.broker.service.EntryBatchIndexesAcks; +import org.apache.pulsar.broker.service.EntryBatchSizes; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; +import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; +import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl; +import org.awaitility.Awaitility; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.Test; + +/** + * Regression test for a single-active-consumer (Failover) subscription that becomes permanently stuck + * when a client redeliver command races the completion of an already-dispatched managed-ledger read. + * + *

Four coupled state variables. A caught-up Failover subscription is described by four pieces + * of state that must stay consistent: + *

    + *
  1. {@code cursor.waitingReadOp} — the armed tail-wait read op + * ({@link ManagedCursorImpl#hasPendingReadRequest()});
    2. + *
  2. the cursor's membership in {@link ManagedLedgerImpl}'s {@code waitingCursors} queue — the + * set a publish walks to wake parked readers;
    3. + *
  3. {@code cursor.pendingReadOps} — reads currently in flight + * ({@link ManagedCursorImpl#getPendingReadOpsCount()});
    4. + *
  4. {@code dispatcher.havePendingRead} — the dispatcher's belief that a read is outstanding.
    5. + *
+ * In healthy operation an armed wait op implies {@code havePendingRead}, and implies the cursor is + * either queued in {@code waitingCursors} or has an in-flight read. + * + *

The race. + * {@code PersistentDispatcherSingleActiveConsumer#internalRedeliverUnacknowledgedMessages} unconditionally + * clears {@code havePendingRead} and arms a fresh read via {@code readMoreEntries} without draining + * a read that {@code notifyEntriesAvailable} has already dispatched. The stale read's completion + * ({@code readEntriesComplete} / {@code readEntriesFailed}) then clears {@code havePendingRead} again: + * there is no read-generation guard on {@code havePendingRead}, and the only staleness check is the + * {@code readConsumer != currentConsumer} identity comparison in {@code readEntriesComplete}, which runs + * after that same method has already cleared the flag at its top. When the redeliver's re-arm and + * the stale completion interleave in the order [redeliver, then stale completion], the result is the seed + * state — an armed wait op with {@code havePendingRead == false}. This is still benign while the + * cursor remains in {@code waitingCursors}. + * + *

The absorbing conversion. When the last consumer disconnects, the durable-cursor path + * ({@code AbstractDispatcherSingleActiveConsumer#removeConsumer} → {@code cancelPendingRead}, then + * {@code ManagedLedger#removeWaitingCursor}) deliberately does not cancel the pending read for a durable + * cursor — see the durable-cursor comment in {@code PersistentSubscription#removeConsumer}. Because + * {@code havePendingRead} is already {@code false}, {@code cancelPendingRead} short-circuits on its + * {@code havePendingRead} guard and leaves the wait op armed, while {@code removeWaitingCursor} strips the + * cursor from the queue. The subscription is now in the absorbing orphan state: an armed wait op, absent + * from {@code waitingCursors}, with no in-flight read and {@code havePendingRead == false}. Every + * subsequent re-arm CAS-fails on the {@code WAITING_READ_OP_UPDATER} compare-and-set in + * {@code ManagedCursorImpl#asyncReadEntriesWithSkipOrWait}, raising {@code ConcurrentWaitCallbackException} + * — which {@code readEntriesFailed} returns early on instead of rescheduling — and every + * publish's {@code notifyCursors} poll-misses. The subscription never reads again — which is exactly + * what both tests assert behaviourally: after the disconnect a fresh consumer reconnects, and a message + * published afterwards must actually reach that consumer's {@code sendMessages}. + * + *

The two racing events genuinely arrive on different threads: the read completion is posted to the + * dispatcher's ordered executor by the managed-ledger/BookKeeper completion chain (the + * {@code entriesFuture.whenCompleteAsync(..., executor)} hand-off at the end of {@code readMoreEntries}), + * while the redeliver is posted by {@code redeliverUnacknowledgedMessages} from the client-command thread. + * Their arrival order is a real race. + * + *

Fidelity notes / test seams. + *

    + *
  • The cursor, ledger, subscription, dispatcher, {@code waitingReadOp} compare-and-set, + * {@code checkForNewEntries}, {@code notifyEntriesAvailable}, {@code notifyCursors}, + * {@code cancelPendingReadRequest}, {@code addWaitingCursor}/{@code removeWaitingCursor} and + * {@code havePendingRead} management are all real production code.
    • + *
  • The consumer lifecycle runs through the real {@code PersistentSubscription#addConsumer} and + * {@code PersistentSubscription#removeConsumer(Consumer, boolean)}, so dispatcher removal, + * {@code deactivateCursor()} and {@code removeWaitingCursor()} happen in the production order. The + * spied dispatcher is handed to the subscription through the existing production test seam + * {@code PersistentSubscription#reuseOrCreateDispatcher}.
    • + *
  • Only the metadata-parsing filter step ({@code filterEntriesForConsumer}) is stubbed on a Mockito + * spy, because the entries this test publishes are raw bytes rather than serialized Pulsar messages. + * The delivery tail itself is real: {@code dispatchEntriesToConsumer} runs and calls + * {@code Consumer#sendMessages}, which the mock consumer records before releasing the entries and + * completing immediately — so the production {@code readMoreEntries} continuation is posted by + * the real listener. Neither seam touches the four state variables above.
    • + *
  • The dispatcher's ordered executor is replaced with a {@link ManualExecutor} (injected through a + * spied {@code getTopicOrderedExecutor()} scoped to dispatcher construction) so every dispatcher + * async hop becomes an explicit task the test drains in a chosen order.
    • + *
  • The consumer's ack of {@code m1} is telescoped: it is applied before the dispatcher delivery of + * {@code m1}'s in-flight read completes. This is what keeps the schedule deterministic — after + * the ack the redeliver's {@code rewind} lands at the tail with no backlog, so the re-arm is a pure + * in-memory tail-wait and no BookKeeper read escapes the {@link ManualExecutor} to race the drain. + * Without the ack the {@code rewind} re-exposes the unacked entry and {@code readMoreEntries} takes + * the immediate-read branch of {@code ManagedCursorImpl#asyncReadEntriesWithSkipOrWait} (the + * {@code hasMoreEntries()} fast path that delegates straight to {@code asyncReadEntriesWithSkip}), + * issuing a real async read whose completion is off-board. The staged state (mark-delete at the + * tail plus an already-dispatched read whose completion is still pending) is production-reachable + * via a redeliver→ack→redeliver sequence.
    • + *
  • {@code newEntriesCheckDelayInMillis} is pinned to {@code 0} so {@code checkForNewEntries} runs + * inline at arm time rather than on the ledger scheduler. This is a determinism pin only; the bug + * itself is not an artifact of the zero delay. With the telescoped ack in place, every + * dispatcher-visible async hop is then an explicit task on the {@link ManualExecutor} board that the + * test drains in a chosen order.
    • + *
+ * + *

Reproduces apache/pulsar#26164. + */ +@Slf4j +public class PersistentDispatcherSingleActiveConsumerStuckReadTest extends MockedBookKeeperTestCase { + + private static final String TOPIC = "persistent://prop/ns/sac-stuck-read"; + private static final int BOARD_WAIT_SECONDS = 5; + private static final long QUIESCE_TIMEOUT_SECONDS = 10; + private static final int QUIESCE_IDLE_ROUNDS = 4; + private static final long DELIVERY_TIMEOUT_SECONDS = 5; + + private PulsarTestContext pulsarTestContext; + private BrokerService brokerService; + + // Real managed-ledger objects under test. + private ManagedLedgerImpl ledger; + private ManagedCursorImpl cursor; + + // Real topic + subscription; the dispatcher is a Mockito spy so we can stub the entry filter step. + private PersistentTopic topic; + private PersistentSubscription subscription; + private PersistentDispatcherSingleActiveConsumer dispatcher; + private Consumer consumer; + + // The dispatcher's ordered executor: a manual task board we drain in a chosen order. + private ManualExecutor dispatcherExecutor; + + // Everything the dispatcher handed to a Consumer#sendMessages tail, in delivery order. + private final List deliveries = Collections.synchronizedList(new ArrayList<>()); + + private final AtomicLong msgCounter = new AtomicLong(); + + private ManagedLedgerConfig initManagedLedgerConfig(ManagedLedgerConfig config) { + // Inline checkForNewEntries: the +10ms task runs synchronously at arm time -> deterministic. + config.setNewEntriesCheckDelayInMillis(0); + config.setMaxEntriesPerLedger(1_000_000); + config.setRetentionTime(1, TimeUnit.HOURS); + config.setRetentionSizeInMB(-1); + return config; + } + + // ----------------------------------------------------------------------------------------------- + // Fixture + // ----------------------------------------------------------------------------------------------- + + /** Build a fresh topic/subscription/dispatcher on top of a real ledger + cursor. */ + private void buildFixture(String ledgerName) throws Exception { + ServiceConfiguration svcConfig = new ServiceConfiguration(); + svcConfig.setBrokerShutdownTimeoutMs(0L); + svcConfig.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); + svcConfig.setClusterName("test"); + svcConfig.setActiveConsumerFailoverDelayTimeMillis(0); + svcConfig.setSystemTopicEnabled(false); + svcConfig.setTopicLevelPoliciesEnabled(false); + + // Real BrokerService via PulsarTestContext, but backed by the real MockedBookKeeper-based + // ManagedLedgerFactory so we open a REAL ledger + cursor. + pulsarTestContext = PulsarTestContext.builderForNonStartableContext() + .config(svcConfig) + .spyByDefault() + .managedLedgerClients(bkc, factory) + .build(); + brokerService = pulsarTestContext.getBrokerService(); + + ledger = (ManagedLedgerImpl) factory.open(ledgerName, initManagedLedgerConfig(new ManagedLedgerConfig())); + topic = new PersistentTopic(TOPIC, ledger, brokerService); + // Open the cursor AFTER topic construction so the topic does not auto-create a second + // subscription sharing this cursor. + cursor = (ManagedCursorImpl) ledger.openCursor("sub"); + // A real subscription that adopts the spied dispatcher through the production test seam, so + // addConsumer/removeConsumer drive the real consumer lifecycle. The returned reference must be + // qualified: the inherited PersistentSubscription.dispatcher field would otherwise shadow ours. + subscription = new PersistentSubscription(topic, "sub", cursor, false) { + @Override + protected Dispatcher reuseOrCreateDispatcher(Dispatcher existingDispatcher, Consumer newConsumer) { + return PersistentDispatcherSingleActiveConsumerStuckReadTest.this.dispatcher; + } + }; + + // Inject the manual dispatcher executor via getTopicOrderedExecutor().chooseThread(), scoped to + // just the dispatcher construction so unrelated broker work keeps using the real executor. + OrderedExecutor realTopicOrdered = brokerService.getTopicOrderedExecutor(); + dispatcherExecutor = new ManualExecutor(); + OrderedExecutor topicOrdered = mock(OrderedExecutor.class); + doReturn(dispatcherExecutor).when(topicOrdered).chooseThread(); + doReturn(dispatcherExecutor).when(topicOrdered).chooseThread(any()); + doReturn(topicOrdered).when(brokerService).getTopicOrderedExecutor(); + dispatcher = spy(new PersistentDispatcherSingleActiveConsumer(cursor, SubType.Failover, -1, topic, + subscription)); + doReturn(realTopicOrdered).when(brokerService).getTopicOrderedExecutor(); + + // The published payloads are raw bytes, not serialized Pulsar messages, so the metadata-parsing + // filter step is stubbed out. The rest of the delivery tail (dispatchEntriesToConsumer -> + // Consumer#sendMessages) is real production code. + doReturn(0).when(dispatcher).filterEntriesForConsumer(any(), any(), any(), any(), any(), + anyBoolean(), any()); + + deliveries.clear(); + consumer = newMockConsumer("c1"); + } + + /** + * A mock consumer whose {@code sendMessages} tail records what the dispatcher delivered, releases the + * entries and completes immediately. The succeeded future notifies its listener inline, so the + * production {@code readMoreEntries} continuation lands on the manual board exactly as it does in + * production. + */ + private Consumer newMockConsumer(String consumerName) { + Consumer mockConsumer = mock(Consumer.class); + doReturn(1000).when(mockConsumer).getAvailablePermits(); + doReturn(1).when(mockConsumer).getAvgMessagesPerEntry(); + doReturn(true).when(mockConsumer).isWritable(); + doReturn(false).when(mockConsumer).readCompacted(); + doReturn(false).when(mockConsumer).isPreciseDispatcherFlowControl(); + doReturn(false).when(mockConsumer).isBlocked(); + doReturn(consumerName).when(mockConsumer).consumerName(); + doReturn(0).when(mockConsumer).getPriorityLevel(); + doReturn(0L).when(mockConsumer).getConsumerEpoch(); + doReturn(SubType.Failover).when(mockConsumer).subType(); + // PersistentSubscription.removeConsumer folds the removed consumer's counters into the subscription. + doReturn(new ConsumerStatsImpl()).when(mockConsumer).getStats(); + doAnswer(inv -> { + List entries = inv.getArgument(0); + for (Entry entry : entries) { + deliveries.add(Delivery.of(consumerName, entry.getPosition())); + entry.release(); + } + EntryBatchSizes batchSizes = inv.getArgument(1); + if (batchSizes != null) { + batchSizes.recyle(); + } + EntryBatchIndexesAcks batchIndexesAcks = inv.getArgument(2); + if (batchIndexesAcks != null) { + batchIndexesAcks.recycle(); + } + return ImmediateEventExecutor.INSTANCE.newSucceededFuture(null); + }).when(mockConsumer).sendMessages(any(), any(), any(), anyInt(), anyLong(), anyLong(), any(), anyLong()); + return mockConsumer; + } + + private void tearDownFixture() { + try { + if (cursor != null && !cursor.isClosed()) { + cursor.close(); + } + } catch (Exception ignore) { + // best-effort cleanup + } + try { + if (ledger != null) { + ledger.close(); + } + } catch (Exception ignore) { + // best-effort cleanup + } + try { + if (pulsarTestContext != null) { + pulsarTestContext.close(); + } + } catch (Exception ignore) { + // best-effort cleanup + } + cursor = null; + ledger = null; + brokerService = null; + pulsarTestContext = null; + } + + @AfterMethod(alwaysRun = true) + public void afterMethod() { + tearDownFixture(); + } + + // ----------------------------------------------------------------------------------------------- + // Manual executor task board + // ----------------------------------------------------------------------------------------------- + + /** A deterministic executor: submitted Runnables queue up and run only when the test drains them. */ + static final class ManualExecutor extends AbstractExecutorService { + private final ArrayDeque queue = new ArrayDeque<>(); + + @Override + public synchronized void execute(Runnable command) { + queue.add(command); + } + + synchronized int size() { + return queue.size(); + } + + /** Remove and return all currently-queued tasks (tasks added later are NOT included). */ + synchronized List takeAll() { + List snapshot = new ArrayList<>(queue); + queue.clear(); + return snapshot; + } + + /** Run one task FIFO; returns false if empty. */ + boolean runOne() { + Runnable r; + synchronized (this) { + r = queue.poll(); + } + if (r == null) { + return false; + } + r.run(); + return true; + } + + @Override + public void shutdown() { + } + + @Override + public List shutdownNow() { + return takeAll(); + } + + @Override + public boolean isShutdown() { + return false; + } + + @Override + public boolean isTerminated() { + return false; + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) { + return true; + } + } + + // ----------------------------------------------------------------------------------------------- + // Operation alphabet (each drives REAL code) + // ----------------------------------------------------------------------------------------------- + + /** Connect a consumer via the REAL PersistentSubscription.addConsumer path; for Failover with a zero + * failover delay this arms the tail-wait read synchronously through the dispatcher's + * scheduleReadOnActiveConsumer -> readMoreEntries. */ + private void connect(Consumer consumerToAdd) throws Exception { + subscription.addConsumer(consumerToAdd).get(); + } + + /** + * Disconnect a consumer via the REAL {@code PersistentSubscription.removeConsumer(consumer, false)} path, + * so dispatcher removal (which calls {@code cancelPendingRead()}), {@code deactivateCursor()} and + * {@code removeWaitingCursor()} all run in the production order. For a durable cursor the subscription + * deliberately does not cancel the pending read — see the durable-cursor comment in + * {@code PersistentSubscription#removeConsumer}. + */ + private void disconnect(Consumer consumerToRemove) throws Exception { + subscription.removeConsumer(consumerToRemove, false); + } + + private Position publish() throws Exception { + return ledger.addEntry(("m" + msgCounter.incrementAndGet()).getBytes()); + } + + private void ackUpTo(Position p) throws Exception { + cursor.markDelete(p); + } + + /** Block (with an explicit timeout) until the dispatcher board holds at least {@code n} tasks. */ + private void awaitBoardHasAtLeast(int n) { + Awaitility.await("dispatcher board should hold at least " + n + " task(s)") + .atMost(Duration.ofSeconds(BOARD_WAIT_SECONDS)) + .pollInterval(Duration.ofMillis(2)) + .until(() -> dispatcherExecutor.size() >= n); + } + + /** + * Drain the dispatcher board until nothing new is produced, with a hard deadline so a regression can + * never hang CI. Async read completions posted by the (real) managed-ledger worker pool land on the + * board between idle rounds. + */ + private void quiesce() { + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(QUIESCE_TIMEOUT_SECONDS); + for (int idleRounds = 0; idleRounds < QUIESCE_IDLE_ROUNDS;) { + if (System.nanoTime() > deadlineNanos) { + throw new IllegalStateException("quiesce() did not settle within " + QUIESCE_TIMEOUT_SECONDS + + "s; board=" + dispatcherExecutor.size() + " pendingReadOps=" + + cursor.getPendingReadOpsCount()); + } + boolean did = false; + while (dispatcherExecutor.runOne()) { + did = true; + } + if (did) { + idleRounds = 0; + continue; + } + if (dispatcherExecutor.size() == 0 && cursor.getPendingReadOpsCount() == 0) { + idleRounds++; + } + sleepQuietly(); + } + } + + /** + * Keep draining the dispatcher board until {@code position} has been handed to {@code target}'s + * {@code sendMessages}, with a hard deadline so a regression fails instead of hanging CI. + */ + private boolean pumpUntilDelivered(Consumer target, Position position) { + Delivery expected = Delivery.of(target.consumerName(), position); + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(DELIVERY_TIMEOUT_SECONDS); + do { + while (dispatcherExecutor.runOne()) { + // A delivery posts follow-up work onto the board; keep draining before re-checking. + } + if (deliveries.contains(expected)) { + return true; + } + sleepQuietly(); + } while (System.nanoTime() < deadlineNanos); + return false; + } + + private static void sleepQuietly() { + try { + Thread.sleep(3); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + // ----------------------------------------------------------------------------------------------- + // State snapshot + predicates + // ----------------------------------------------------------------------------------------------- + + /** One entry handed to a consumer by the dispatcher's real delivery tail. */ + private record Delivery(String consumerName, long ledgerId, long entryId) { + + static Delivery of(String consumerName, Position position) { + return new Delivery(consumerName, position.getLedgerId(), position.getEntryId()); + } + + @Override + public String toString() { + return consumerName + "@" + ledgerId + ":" + entryId; + } + } + + /** + * A snapshot of the four coupled state variables. {@code inQueue} is derived from + * {@code ledger.getWaitingCursorsCount()}; the single-cursor fixture makes that ledger-wide count a + * per-cursor membership proxy (0 ⇔ this cursor is not queued). + */ + private record StuckState(boolean armed, boolean inQueue, int pendingReadOps, boolean havePendingRead) { + + /** The absorbing orphan state: armed, not queued, no in-flight read, and havePendingRead cleared. */ + boolean stranded() { + return armed && !inQueue && pendingReadOps == 0 && !havePendingRead; + } + + /** The benign seed: armed with havePendingRead cleared, but still queued (so not yet stranded). */ + boolean seed() { + return armed && inQueue && !havePendingRead; + } + + @Override + public String toString() { + return "armed=" + armed + " inQueue=" + inQueue + " pendingReadOps=" + pendingReadOps + + " havePendingRead=" + havePendingRead; + } + } + + private StuckState snapshot() { + return new StuckState(cursor.hasPendingReadRequest(), ledger.getWaitingCursorsCount() >= 1, + cursor.getPendingReadOpsCount(), dispatcher.havePendingRead); + } + + /** + * Tail shared by both schedules, asserting the externally observable contract of #26164: after the last + * consumer disconnects and a fresh consumer reconnects to the same durable subscription, a message + * published afterwards must actually be delivered to that consumer. On buggy master the primary + * schedule leaves the cursor in the absorbing orphan state, the reconnect's re-arm CAS-fails and the + * message never reaches {@code sendMessages}. + */ + private void assertSubscriptionDeliversAfterReconnect(String schedule, StuckState preDisconnect) + throws Exception { + disconnect(consumer); + quiesce(); + StuckState postDisconnect = snapshot(); + log.info("post-disconnect quiescent state state={}", postDisconnect); + + // A client reconnect: a brand new consumer takes over the (still durable) subscription. + Consumer reconnected = newMockConsumer("c2"); + connect(reconnected); + quiesce(); + StuckState postReconnect = snapshot(); + log.info("post-reconnect quiescent state state={}", postReconnect); + + Position m2 = publish(); + boolean deliveredM2 = pumpUntilDelivered(reconnected, m2); + quiesce(); + StuckState afterWake = snapshot(); + log.info("post-wake-publish state state={} delivered={}", afterWake, deliveredM2); + + String states = "schedule=[" + schedule + "] pre-disconnect=[" + preDisconnect + "] post-disconnect=[" + + postDisconnect + "] post-reconnect=[" + postReconnect + "] after-wake-publish=[" + afterWake + + "] deliveries=" + deliveries; + + // Primary, behavioural: the reconnected consumer must actually receive the new message. + Assert.assertTrue(deliveredM2, + "BUG REPRODUCED: after the last consumer disconnected and a new one reconnected, the message " + + "published next (" + m2 + ") never reached the reconnected consumer's sendMessages within " + + DELIVERY_TIMEOUT_SECONDS + "s; the Failover subscription is permanently stuck. " + states); + Assert.assertTrue(cursor.getReadPosition().compareTo(m2) > 0, + "the cursor read position must have advanced past the delivered message " + m2 + ", but it is " + + cursor.getReadPosition() + ". " + states); + + // Secondary, diagnostic: name the internal tuple that causes the stall, so a failure points at it. + Assert.assertFalse(afterWake.stranded(), + "the cursor must not be left in the absorbing orphan state (armed waitingReadOp, absent from " + + "ManagedLedger.waitingCursors, no in-flight read, havePendingRead=false). " + states); + } + + /** Stage the shared precondition: an armed tail-wait read plus a read completion pending on the board. */ + private void stageInFlightReadCompletion() throws Exception { + // Arm the tail-wait read: addConsumer -> scheduleReadOnActiveConsumer -> readMoreEntries. + connect(consumer); + Assert.assertTrue(cursor.hasPendingReadRequest(), "the tail-wait read op should be armed after addConsumer"); + Assert.assertEquals(ledger.getWaitingCursorsCount(), 1, "the cursor should be registered in waitingCursors"); + Assert.assertTrue(dispatcher.havePendingRead, "havePendingRead should be true while the tail read is armed"); + + // Publish m1: notifyCursors -> notifyEntriesAvailable dispatches the armed op's read, advancing the + // read position past m1 and posting readEntriesComplete onto the board. The wait op is now consumed, + // so hasPendingReadRequest() is false, but havePendingRead is still true. + Position m1 = publish(); + awaitBoardHasAtLeast(1); + Assert.assertFalse(cursor.hasPendingReadRequest(), + "the armed op should have been consumed by notifyEntriesAvailable"); + + // Telescoped ack: mark-delete m1 before its already-dispatched read completes. This lands the + // redeliver's subsequent rewind at the tail with no backlog, so the re-arm is a pure in-memory + // tail-wait and no BookKeeper read escapes the ManualExecutor to race quiesce(). The staged state + // (mark-delete at tail + an already-dispatched read whose completion is still pending) is + // production-reachable via a redeliver->ack->redeliver sequence (see the class Javadoc). + ackUpTo(m1); + + // Redeliver: posts internalRedeliver onto the board, behind the still-pending read completion. + dispatcher.redeliverUnacknowledgedMessages(consumer, 1L); + awaitBoardHasAtLeast(2); + } + + /** + * The board now holds exactly two tasks from two different producers: + *

    + *
  1. {@code readEntriesComplete} — posted by the managed-ledger/BookKeeper completion chain + * ({@code whenCompleteAsync});
    2. + *
  2. {@code internalRedeliver} — posted by {@code redeliverUnacknowledgedMessages} (the + * client-command thread).
    3. + *
+ */ + private List takeTwoRacingTasks() { + List board = dispatcherExecutor.takeAll(); + Assert.assertEquals(board.size(), 2, + "the board must hold exactly two tasks from two producers: the read completion " + + "(posted by the managed-ledger/BookKeeper completion chain) and internalRedeliver " + + "(posted by the client-command thread)"); + return board; + } + + // =============================================================================================== + // (a) PRIMARY reproduction: a redeliver races the completion of an already-dispatched read. m1's ack is + // telescoped (applied before that read completes) so the redeliver's rewind lands at the tail and the + // re-arm is a pure in-memory tail-wait -- fully deterministic. See the class Javadoc fidelity notes. + // =============================================================================================== + + @Test(groups = "broker") + public void testFailoverConsumerStuckWhenRedeliverRacesInFlightReadCompletion() throws Exception { + buildFixture("sac-stuck-read-race"); + stageInFlightReadCompletion(); + List board = takeTwoRacingTasks(); + + // Critical interleaving: run the redeliver's re-arm BEFORE the stale read completion. internalRedeliver + // clears havePendingRead and arms a fresh read; the stale completion then clears havePendingRead again + // while the fresh op stays armed -> the seed state. + board.get(1).run(); // internalRedeliver + board.get(0).run(); // stale readEntriesComplete + quiesce(); + + // Observation only (NOT asserted): the race mints the seed (armed op + havePendingRead=false, still + // queued). The seed is deliberately not asserted -- a completion-side fix (a read-generation guard on + // havePendingRead) would prevent the seed from ever forming, so binding on it would couple this test to + // one fix strategy. The binding assertions are the behavioural ones after the disconnect + reconnect. + StuckState seed = snapshot(); + log.info("post-redeliver-race quiescent state (observation only) state={} seedPreconditionHeld={}", + seed, seed.seed()); + + assertSubscriptionDeliversAfterReconnect("redeliver, then stale completion", seed); + } + + // =============================================================================================== + // (b) NEGATIVE CONTROL: the same setup but the benign FIFO order [stale completion, then redeliver]. + // The invariant holds end to end; this PASSES on master. + // =============================================================================================== + + @Test(groups = "broker") + public void testRedeliverAfterReadCompletionDoesNotStrandCursor() throws Exception { + buildFixture("sac-stuck-read-fifo"); + stageInFlightReadCompletion(); + List board = takeTwoRacingTasks(); + + // Benign FIFO order: the stale completion runs FIRST (clearing havePendingRead), then the redeliver + // re-arms and sets havePendingRead=true. The armed op and havePendingRead stay coupled, so no seed + // forms and the later disconnect can cancel the op cleanly. + board.get(0).run(); // stale readEntriesComplete + board.get(1).run(); // internalRedeliver + quiesce(); + + StuckState healthy = snapshot(); + log.info("post-FIFO-race quiescent state state={}", healthy); + Assert.assertFalse(healthy.stranded(), "the benign FIFO order must not strand the cursor: " + healthy); + + // The disconnect now finds havePendingRead=true, so cancelPendingRead actually cancels the armed op: + // no orphan is created, the reconnect re-arms cleanly and the next publish is delivered. + assertSubscriptionDeliversAfterReconnect("stale completion, then redeliver (negative control)", healthy); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumerTest.java index 466f55436ea40..10295d06dac87 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumerTest.java @@ -119,7 +119,7 @@ public void testSkipReadEntriesFromCloseCursor() throws Exception { Mockito.doAnswer(inv -> { callReadEntriesFailed.getAndIncrement(); return inv.callRealMethod(); - }).when(dispatcher).readEntriesFailed(Mockito.any(), Mockito.any()); + }).when(dispatcher).readEntriesFailed(Mockito.any(), Mockito.any(), Mockito.anyLong()); Mockito.doReturn(false).when(cursor).isClosed(); From 67b3fb854d92fc2ea843d889c988126e865071e3 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Sat, 25 Jul 2026 13:57:02 +0300 Subject: [PATCH 139/213] [fix][broker] Fix Key_Shared delivery stall when look-ahead triggers at the end of the topic (#26236) (cherry picked from commit 37ceb98fb0a2feecd8eb295207edcee30049c0ba) (cherry picked from commit f066cfe8a09e991ee153d2fb90f1148e2a4386da) --- ...tStickyKeyDispatcherMultipleConsumers.java | 13 ++- ...ckyKeyDispatcherMultipleConsumersTest.java | 99 +++++++++++++++++++ 2 files changed, 111 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java index 8a54d07f230aa..436e7c43e010f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java @@ -316,13 +316,24 @@ protected synchronized boolean trySendMessagesToConsumers(ReadType readType, Lis acquirePermitsForDeliveredMessages(topic, cursor, totalEntries, totalMessagesSent, totalBytesSent); // trigger read more messages if necessary - if (triggerLookAhead.booleanValue()) { + if (triggerLookAhead.booleanValue() && (allowOutOfOrderDelivery || cursor.hasMoreEntries())) { // When all messages get filtered and no messages are sent, we should read more entries, "look ahead" // so that a possible next batch of messages might contain messages that can be dispatched. // This is done only when there's a consumer with available permits, and it's not able to make progress // because of blocked hashes. Without this rule we would be looking ahead in the stream while the // new consumers are not ready to accept the new messages, // therefore would be most likely only increase the distance between read-position and mark-delete position. + // When ordered delivery is required, look-ahead is engaged only when the cursor has more entries. + // Otherwise the next readMoreEntries call would skip replaying the replay queue and pulling due messages + // from the delayed delivery tracker, and instead issue a normal read that waits at the end of the topic + // for new entries. That would leave deliverable messages stuck in the replay queue or the delayed + // delivery tracker until an unrelated event (such as a consumer flow request) triggers another read, + // stalling dispatch (issue #21554). + // When out-of-order delivery is allowed, look-ahead is engaged unconditionally, as before. In that mode + // the replay queue doesn't track sticky key hashes, so the replay position filter cannot exclude + // messages for consumers without available permits, and each replay would re-read and discard the same + // undispatchable messages. Ending the cycle with a look-ahead attempt (that waits at the end of the + // topic when there is nothing to read) prevents such repeated read-and-discard loops. skipNextReplayToTriggerLookAhead = true; // skip backoff delay before reading ahead in the "look ahead" mode to prevent any additional latency // only skip the delay if there are more entries to read diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java index 26e16616d1add..50007b5c86197 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java @@ -463,6 +463,105 @@ public void testSkipRedeliverTemporally() throws InterruptedException { allEntries.forEach(Entry::release); } + /** + * Reproduces the dispatch stall behind the flaky + * KeySharedSubscriptionTest.testContinueDispatchMessagesWhenMessageDelayed (issue #21554). + * + * When a replay read gets fully discarded (for example because the target consumer ran out of permits while + * the read was in flight) and the cursor has no more entries, engaging the "look ahead" mode made the follow-up + * read skip the replay queue and issue a normal read that waits at the end of the topic for new entries. Messages + * in the replay queue for consumers with available permits were then stuck until an unrelated event, such as a + * consumer flow request, triggered another read. + */ + @Test(timeOut = 30000) + public void testLookAheadNotEngagedWhenCursorHasNoMoreEntries() throws Exception { + persistentDispatcher.close(); + + // the mocked executor doesn't support schedule(), so run the rescheduled read directly + persistentDispatcher = new PersistentStickyKeyDispatcherMultipleConsumers( + topicMock, cursorMock, subscriptionMock, configMock, + new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT)) { + @Override + protected void reScheduleReadInMs(long readAfterMs) { + orderedExecutor.execute(this::readMoreEntries); + } + }; + + // consumer1 has available permits at all times + persistentDispatcher.addConsumer(consumerMock).join(); + + // the slow consumer initially has 2 permits + final Consumer slowConsumerMock = createMockConsumer(); + doReturn("consumer2").when(slowConsumerMock).consumerName(); + doReturn(true).when(slowConsumerMock).isWritable(); + AtomicInteger slowConsumerAvailablePermits = new AtomicInteger(2); + doAnswer(invocation -> slowConsumerAvailablePermits.get()).when(slowConsumerMock).getAvailablePermits(); + persistentDispatcher.addConsumer(slowConsumerMock).join(); + + StickyKeyConsumerSelector selector = persistentDispatcher.getSelector(); + String keyForConsumer1 = generateKeyForConsumer(selector, consumerMock); + String keyForSlowConsumer = generateKeyForConsumer(selector, slowConsumerMock); + + final Entry entry1 = createEntry(1, 1, "message1", 1, keyForSlowConsumer); + final Entry entry2 = createEntry(1, 2, "message2", 2, keyForSlowConsumer); + final Entry entry3 = createEntry(1, 3, "message3", 3, keyForConsumer1); + final List allEntries = List.of(entry1, entry2, entry3); + + // the cursor has no more entries; a normal read would wait for new entries without completing + doReturn(false).when(cursorMock).hasMoreEntries(); + doAnswer(invocationOnMock -> null) + .when(cursorMock).asyncReadEntriesWithSkipOrWait(anyInt(), anyLong(), any(), any(), any(), any()); + + // Mock Cursor#asyncReplayEntries. While the first replay read is in flight, the slow consumer runs out of + // permits and a message for consumer1 becomes replayable. This mirrors the delayed delivery tracker feeding + // messages to the replay queue while dispatching is in progress. + doAnswer(invocationOnMock -> { + Set positionsArg = invocationOnMock.getArgument(0); + Set positions = new TreeSet<>(positionsArg); + if (!positions.contains(entry3.getPosition())) { + slowConsumerAvailablePermits.set(0); + // add extra retain since addEntryToReplay will release it + ((EntryImpl) entry3).retain(); + persistentDispatcher.addEntryToReplay(entry3); + } + List entries = allEntries.stream() + .filter(entry -> positions.contains(entry.getPosition())) + .toList(); + AsyncCallbacks.ReadEntriesCallback callback = invocationOnMock.getArgument(1); + Object ctx = invocationOnMock.getArgument(2); + callback.readEntriesComplete(copyEntries(entries), ctx); + return Collections.emptySet(); + }).when(cursorMock).asyncReplayEntries(anySet(), any(), any(), anyBoolean()); + + CountDownLatch consumer1ReceivedMessage3 = new CountDownLatch(1); + mockSendMessages(consumerMock, entries -> { + boolean message3Found = entries.stream() + .anyMatch(entry -> entry.getPosition().equals(entry3.getPosition())); + if (message3Found) { + consumer1ReceivedMessage3.countDown(); + } + }); + mockSendMessages(slowConsumerMock, entries -> { }); + + // seed the replay queue with the slow consumer's entries + for (Entry entry : List.of(entry1, entry2)) { + // add extra retain since addEntryToReplay will release it + ((EntryImpl) entry).retain(); + persistentDispatcher.addEntryToReplay(entry); + } + + // trigger the replay read. The batch gets fully discarded since the slow consumer runs out of permits while + // the read is in flight. The message for consumer1 that was added to the replay queue in the meantime must + // get dispatched by the follow-up read instead of the dispatcher parking a normal read at the end of the + // topic. + persistentDispatcher.readMoreEntries(); + + assertTrue(consumer1ReceivedMessage3.await(5, TimeUnit.SECONDS), + "The replayed message for consumer1 with available permits should have been dispatched"); + + allEntries.forEach(Entry::release); + } + @Test(timeOut = 30000) public void testMessageRedelivery() throws Exception { final List actualEntriesToConsumer1 = new CopyOnWriteArrayList<>(); From 320389cec5db9059e8bfabc309d1d325bbe6e718 Mon Sep 17 00:00:00 2001 From: void-ptr974 Date: Sat, 25 Jul 2026 22:50:08 +0800 Subject: [PATCH 140/213] [fix][broker] Prevent stale topic unload cleanup from removing active cache entries (#26145) (cherry picked from commit 73549d246541dd3d12bd6e17e246cddda5e436d8) Signed-off-by: Zixuan Liu --- .../pulsar/broker/service/BrokerService.java | 112 ++++++--- .../broker/service/BrokerServiceTest.java | 213 +++++++++++++++++- 2 files changed, 287 insertions(+), 38 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 5fabf6c10e3e4..30b2f439d603e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -232,6 +232,11 @@ public class BrokerService implements Closeable { private final Map>> topics = new ConcurrentHashMap<>(); + // A topic can reach the cache-cleanup path through both its close callback and bundle cleanup. + // Keep only one cleanup active for a topic future so re-entrant unload listeners cannot duplicate side effects. + private final Set>> topicCacheRemovalsInProgress = + ConcurrentHashMap.newKeySet(); + private final Map replicationClients = new ConcurrentHashMap<>(); private final Map clusterAdmins = new ConcurrentHashMap<>(); @@ -2619,13 +2624,19 @@ private CompletableFuture unloadServiceUnit(NamespaceBundle serviceUnit } public void cleanUnloadedTopicFromCache(NamespaceBundle serviceUnit) { - for (String topic : topics.keySet()) { + topics.forEach((topic, topicFuture) -> { TopicName topicName = TopicName.get(topic); - if (serviceUnit.includes(topicName) && getTopicReference(topic).isPresent()) { + Optional topicRef = extractTopic(topicFuture); + // The bundle cleanup should only remove topics already fenced by close/unload. + if (serviceUnit.includes(topicName) && topicRef.isPresent() && isTopicBeingUnloaded(topicRef.get())) { log.info("[{}][{}] Clean unloaded topic from cache.", serviceUnit.toString(), topic); - pulsar.getBrokerService().removeTopicFromCache(topicName.toString(), serviceUnit, null); + removeTopicFromCache(topicName.toString(), serviceUnit, topicFuture); } - } + }); + } + + private boolean isTopicBeingUnloaded(Topic topic) { + return topic instanceof AbstractTopic abstractTopic && abstractTopic.isFenced(); } public AuthorizationService getAuthorizationService() { @@ -2634,7 +2645,8 @@ public AuthorizationService getAuthorizationService() { /** * Removes the topic from the cache only if the topicName and associated createFuture match exactly. - * The TopicEvent.UNLOAD event will be triggered before and after removal. + * The TopicEvent.UNLOAD BEFORE event is triggered while the matching cache entry is still present; + * the SUCCESS event is triggered after it has been removed. * * @param topic The topic to be removed. * @return A CompletableFuture that completes when the operation is done. @@ -2649,45 +2661,71 @@ public CompletableFuture removeTopicFromCache(AbstractTopic topic) { private void removeTopicFromCache(String topic, NamespaceBundle namespaceBundle, CompletableFuture> createTopicFuture) { - String bundleName = namespaceBundle.toString(); - String namespaceName = TopicName.get(topic).getNamespaceObject().toString(); - - topicEventsDispatcher.newEvent(topic, TopicEvent.UNLOAD).stage(EventStage.BEFORE).dispatch(); - - synchronized (multiLayerTopicsMap) { - final var namespaceMap = multiLayerTopicsMap.get(namespaceName); - if (namespaceMap != null) { - final var bundleMap = namespaceMap.get(bundleName); - if (bundleMap != null) { - bundleMap.remove(topic); - if (bundleMap.isEmpty()) { - namespaceMap.remove(bundleName); - } + if (createTopicFuture == null) { + if (log.isDebugEnabled()) { + log.debug("[{}] Skip removing topic from cache without its expected future.", topic); + } + return; + } + if (!topicCacheRemovalsInProgress.add(createTopicFuture)) { + if (log.isDebugEnabled()) { + log.debug("[{}] Skip removing topic from cache since cleanup is already in progress.", topic); + } + return; + } + try { + if (topics.get(topic) != createTopicFuture) { + // A stale close/unload callback must not emit unload side effects for a superseded topic future. + if (log.isDebugEnabled()) { + log.debug("[{}] Skip removing topic from cache since it was already removed or superseded.", + topic); } + return; + } + String bundleName = namespaceBundle.toString(); + String namespaceName = TopicName.get(topic).getNamespaceObject().toString(); + + topicEventsDispatcher.newEvent(topic, TopicEvent.UNLOAD).stage(EventStage.BEFORE).dispatch(); + + synchronized (multiLayerTopicsMap) { + final var namespaceMap = multiLayerTopicsMap.get(namespaceName); + if (namespaceMap != null) { + final var bundleMap = namespaceMap.get(bundleName); + if (bundleMap != null) { + bundleMap.remove(topic); + if (bundleMap.isEmpty()) { + namespaceMap.remove(bundleName); + } + } - if (namespaceMap.isEmpty()) { - multiLayerTopicsMap.remove(namespaceName); - final ClusterReplicationMetrics clusterReplicationMetrics = pulsarStats - .getClusterReplicationMetrics(); - replicationClients.forEach((cluster, client) -> { - clusterReplicationMetrics.remove(clusterReplicationMetrics.getKeyName(namespaceName, - cluster)); - }); + if (namespaceMap.isEmpty()) { + multiLayerTopicsMap.remove(namespaceName); + final ClusterReplicationMetrics clusterReplicationMetrics = pulsarStats + .getClusterReplicationMetrics(); + replicationClients.forEach((cluster, client) -> { + clusterReplicationMetrics.remove(clusterReplicationMetrics.getKeyName(namespaceName, + cluster)); + }); + } } } - } - if (createTopicFuture == null) { - topics.remove(topic); - } else { - topics.remove(topic, createTopicFuture); - } + Compactor compactor = pulsar.getNullableCompactor(); + if (compactor != null) { + compactor.getStats().removeTopic(topic); + } - Compactor compactor = pulsar.getNullableCompactor(); - if (compactor != null) { - compactor.getStats().removeTopic(topic); + if (!topics.remove(topic, createTopicFuture)) { + if (log.isDebugEnabled()) { + log.debug("[{}] Skip unload success because the cache entry was already removed or superseded.", + topic); + } + return; + } + topicEventsDispatcher.notify(topic, TopicEvent.UNLOAD, EventStage.SUCCESS); + } finally { + topicCacheRemovalsInProgress.remove(createTopicFuture); } - topicEventsDispatcher.newEvent(topic, TopicEvent.UNLOAD).dispatch(); } public long getNumberOfNamespaceBundles() { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java index 96fb38e62620a..0d0e32f0d4f90 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java @@ -1327,6 +1327,187 @@ public void testCheckInactiveSubscriptionsShouldNotDeleteCompactionCursor() thro } + @Test + public void testCleanUnloadedTopicFromCacheDoesNotRemoveNewTopicFuture() throws Exception { + final String namespace = "prop/ns-abc"; + final String topicName = "persistent://" + namespace + "/cleanUnloadedTopicFromCache-" + + UUID.randomUUID(); + final BrokerService brokerService = pulsar.getBrokerService(); + Producer producer = pulsarClient.newProducer().topic(topicName).create(); + producer.close(); + Topic topic = brokerService.getTopicReference(topicName).orElseThrow(); + CompletableFuture> originalTopicFuture = brokerService.getTopics().get(topicName); + assertNotNull(originalTopicFuture); + NamespaceBundle bundle = pulsar.getNamespaceService().getBundle(TopicName.get(topicName)); + BlockingCompletedFuture> oldTopicFuture = + new BlockingCompletedFuture<>(Optional.of(topic)); + brokerService.getTopics().put(topicName, oldTopicFuture); + ((AbstractTopic) topic).isFenced = true; + + CompletableFuture> newTopicFuture = + CompletableFuture.completedFuture(Optional.of(mock(Topic.class))); + List unloadEvents = new ArrayList<>(); + TopicEventsListener listener = (name, event, stage, t) -> { + if (topicName.equals(name) && event == TopicEventsListener.TopicEvent.UNLOAD) { + unloadEvents.add(stage); + } + }; + brokerService.addTopicEventListener(listener); + try { + CompletableFuture cleanupFuture = CompletableFuture.runAsync( + () -> brokerService.cleanUnloadedTopicFromCache(bundle)); + assertTrue(oldTopicFuture.awaitIsDone(5, TimeUnit.SECONDS), + "cleanup should capture the old future before it is replaced"); + + // Simulate a reload racing after the bundle cleanup has captured the old future. + brokerService.getTopics().put(topicName, newTopicFuture); + oldTopicFuture.allowIsDone(); + cleanupFuture.get(5, TimeUnit.SECONDS); + + assertTrue(brokerService.getTopics().get(topicName) == newTopicFuture, + "A stale bundle unload cleanup must not remove a newer topic future"); + assertTrue(unloadEvents.isEmpty(), + "A stale bundle unload cleanup must not dispatch unload events"); + } finally { + oldTopicFuture.allowIsDone(); + brokerService.removeTopicEventListener(listener); + brokerService.getTopics().put(topicName, originalTopicFuture); + topic.close(false, true).get(5, TimeUnit.SECONDS); + } + } + + @Test + public void testUnloadBeforeDoesNotStartTopicReload() throws Exception { + final String namespace = "prop/ns-abc"; + final String topicName = "persistent://" + namespace + "/unloadBeforeDoesNotStartTopicReload-" + + UUID.randomUUID(); + final BrokerService brokerService = pulsar.getBrokerService(); + Producer producer = pulsarClient.newProducer().topic(topicName).create(); + producer.close(); + Topic topic = brokerService.getTopicReference(topicName).orElseThrow(); + CompletableFuture> topicFuture = brokerService.getTopics().get(topicName); + assertNotNull(topicFuture); + + AtomicReference>> futureAtUnloadBefore = new AtomicReference<>(); + TopicEventsListener listener = (name, event, stage, t) -> { + if (topicName.equals(name) && event == TopicEventsListener.TopicEvent.UNLOAD) { + if (stage == TopicEventsListener.EventStage.BEFORE) { + futureAtUnloadBefore.set(brokerService.getTopic(TopicName.get(topicName), true, null)); + } + } + }; + brokerService.addTopicEventListener(listener); + try { + topic.close(true, true).get(5, TimeUnit.SECONDS); + + assertTrue(futureAtUnloadBefore.get() == topicFuture, + "UNLOAD BEFORE must return the existing topic future instead of starting a reload"); + assertNull(brokerService.getTopics().get(topicName), + "UNLOAD SUCCESS must observe the topic cache entry removed"); + } finally { + brokerService.removeTopicEventListener(listener); + } + } + + @Test + public void testRemoveTopicFromCacheIgnoresReentrantUnloadCallback() throws Exception { + final String namespace = "prop/ns-abc"; + final String topicName = "persistent://" + namespace + "/removeTopicFromCacheReentrant-" + + UUID.randomUUID(); + final BrokerService brokerService = pulsar.getBrokerService(); + Producer producer = pulsarClient.newProducer().topic(topicName).create(); + producer.close(); + Topic topic = brokerService.getTopicReference(topicName).orElseThrow(); + + List unloadEvents = new ArrayList<>(); + AtomicReference reentrantCallbackError = new AtomicReference<>(); + TopicEventsListener listener = (name, event, stage, t) -> { + if (topicName.equals(name) && event == TopicEventsListener.TopicEvent.UNLOAD) { + unloadEvents.add(stage); + if (stage == TopicEventsListener.EventStage.BEFORE) { + try { + brokerService.removeTopicFromCache((AbstractTopic) topic).get(5, TimeUnit.SECONDS); + } catch (Throwable e) { + reentrantCallbackError.set(e); + } + } + } + }; + brokerService.addTopicEventListener(listener); + try { + brokerService.removeTopicFromCache((AbstractTopic) topic).get(5, TimeUnit.SECONDS); + + assertNull(reentrantCallbackError.get(), "A reentrant cache cleanup should complete without error"); + assertEquals(unloadEvents, List.of(TopicEventsListener.EventStage.BEFORE, + TopicEventsListener.EventStage.SUCCESS)); + } finally { + brokerService.removeTopicEventListener(listener); + topic.close(false, true).get(5, TimeUnit.SECONDS); + } + } + + @Test + public void testRemoveTopicFromCacheDoesNotRemoveSupersededTopicFuture() throws Exception { + final String namespace = "prop/ns-abc"; + final String topicName = "persistent://" + namespace + "/removeSupersededTopicFromCache-" + + UUID.randomUUID(); + final BrokerService brokerService = pulsar.getBrokerService(); + Producer producer = pulsarClient.newProducer().topic(topicName).create(); + producer.close(); + Topic topic = brokerService.getTopicReference(topicName).orElseThrow(); + CompletableFuture> oldTopicFuture = brokerService.getTopics().get(topicName); + assertNotNull(oldTopicFuture); + + List unloadEvents = new ArrayList<>(); + TopicEventsListener listener = (name, event, stage, t) -> { + if (topicName.equals(name) && event == TopicEventsListener.TopicEvent.UNLOAD) { + unloadEvents.add(stage); + } + }; + brokerService.addTopicEventListener(listener); + + CompletableFuture> newTopicFuture = + CompletableFuture.completedFuture(Optional.of(topic)); + // Simulate the same topic being reloaded before the old topic close callback removes it. + brokerService.getTopics().put(topicName, newTopicFuture); + try { + brokerService.removeTopicFromCache((AbstractTopic) topic).get(5, TimeUnit.SECONDS); + + assertTrue(brokerService.getTopics().get(topicName) == newTopicFuture, + "A stale topic close callback must not remove a newer topic future"); + assertTrue(unloadEvents.isEmpty(), + "A stale topic close callback must not dispatch unload events"); + } finally { + brokerService.removeTopicEventListener(listener); + brokerService.getTopics().put(topicName, oldTopicFuture); + brokerService.removeTopicFromCache((AbstractTopic) topic).get(5, TimeUnit.SECONDS); + } + } + + @Test + public void testCleanUnloadedTopicFromCacheRemovesInactiveBundleTopicFuture() throws Exception { + final String namespace = "prop/ns-abc"; + final String topicName = "persistent://" + namespace + "/cleanInactiveUnloadedTopicFromCache-" + + UUID.randomUUID(); + final BrokerService brokerService = pulsar.getBrokerService(); + Producer producer = pulsarClient.newProducer().topic(topicName).create(); + producer.close(); + Topic topic = brokerService.getTopicReference(topicName).orElseThrow(); + NamespaceBundle bundle = pulsar.getNamespaceService().getBundle(TopicName.get(topicName)); + + pulsar.getNamespaceService().getOwnershipCache().updateBundleState(bundle, false).get(5, TimeUnit.SECONDS); + ((AbstractTopic) topic).isFenced = true; + try { + brokerService.cleanUnloadedTopicFromCache(bundle); + + assertFalse(brokerService.getTopics().containsKey(topicName), + "Inactive bundle cleanup should remove the matching topic future"); + } finally { + pulsar.getNamespaceService().getOwnershipCache().updateBundleState(bundle, true).get(5, TimeUnit.SECONDS); + topic.close(false, true).get(5, TimeUnit.SECONDS); + } + } + @Test public void testCheckInactiveSubscriptionWhenNoMessageToAck() throws Exception { String namespace = "prop/testInactiveSubscriptionWhenNoMessageToAck"; @@ -2049,6 +2230,37 @@ public void testGetTopicWhenTopicPoliciesFail() throws Exception { assertFalse(MockTopicPoliciesService.FAILED_TOPICS.contains(topicName)); } + private static class BlockingCompletedFuture extends CompletableFuture { + private final CountDownLatch isDoneEntered = new CountDownLatch(1); + private final CountDownLatch allowIsDone = new CountDownLatch(1); + + BlockingCompletedFuture(T value) { + complete(value); + } + + @Override + public boolean isDone() { + isDoneEntered.countDown(); + try { + if (!allowIsDone.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("Timed out waiting to resume topic-cache cleanup"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError("Interrupted while waiting to resume topic-cache cleanup", e); + } + return super.isDone(); + } + + boolean awaitIsDone(long timeout, TimeUnit unit) throws InterruptedException { + return isDoneEntered.await(timeout, unit); + } + + void allowIsDone() { + allowIsDone.countDown(); + } + } + static class MockTopicPoliciesService extends TopicPoliciesService.TopicPoliciesServiceDisabled { static final Set FAILED_TOPICS = ConcurrentHashMap.newKeySet(); @@ -2064,4 +2276,3 @@ public CompletableFuture> getTopicPoliciesAsync(TopicNam } } } - From 8f67fc7be34d92798c44b00b2db941090db3eb11 Mon Sep 17 00:00:00 2001 From: sinan liu Date: Sat, 25 Jul 2026 23:18:20 +0800 Subject: [PATCH 141/213] [fix][broker] Prevent completing replicated snapshot before marker publish (#26119) (cherry picked from commit bc0e8b557915f00538200185858b10a161ad8a03) --- .../ReplicatedSubscriptionsController.java | 52 +++++-- ...eplicatedSubscriptionsSnapshotBuilder.java | 25 +++- ...ReplicatedSubscriptionsControllerTest.java | 140 ++++++++++++++++++ ...catedSubscriptionsSnapshotBuilderTest.java | 9 ++ 4 files changed, 214 insertions(+), 12 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java index f19570afcd6e8..d14cd1d2fe73d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.Optional; import java.util.TreeMap; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ScheduledFuture; @@ -289,6 +290,10 @@ private void cleanupTimedOutSnapshots() { while (it.hasNext()) { Map.Entry entry = it.next(); if (entry.getValue().isTimedOut()) { + if (!pendingSnapshots.remove(entry.getKey(), entry.getValue())) { + continue; + } + if (log.isDebugEnabled()) { log.debug("[{}] Snapshot creation timed out for {}", topic.getName(), entry.getKey()); } @@ -297,23 +302,23 @@ private void cleanupTimedOutSnapshots() { timedoutSnapshotsMetric.inc(); var latencyMillis = entry.getValue().getDurationMillis(); stats.recordSnapshotTimedOut(latencyMillis); - it.remove(); } } } void snapshotCompleted(String snapshotId) { ReplicatedSubscriptionsSnapshotBuilder snapshot = pendingSnapshots.remove(snapshotId); - lastCompletedSnapshotId = snapshotId; + if (snapshot == null) { + return; + } - if (snapshot != null) { - lastCompletedSnapshotStartTime = snapshot.getStartTimeMillis(); + lastCompletedSnapshotId = snapshotId; + lastCompletedSnapshotStartTime = snapshot.getStartTimeMillis(); - pendingSnapshotsMetric.dec(); - var latencyMillis = snapshot.getDurationMillis(); - ReplicatedSubscriptionsSnapshotBuilder.SNAPSHOT_METRIC.observe(latencyMillis); - stats.recordSnapshotCompleted(latencyMillis); - } + pendingSnapshotsMetric.dec(); + var latencyMillis = snapshot.getDurationMillis(); + ReplicatedSubscriptionsSnapshotBuilder.SNAPSHOT_METRIC.observe(latencyMillis); + stats.recordSnapshotCompleted(latencyMillis); } void writeMarker(ByteBuf marker) { @@ -324,6 +329,35 @@ void writeMarker(ByteBuf marker) { } } + CompletableFuture writeMarkerAndGetPosition(ByteBuf marker) { + CompletableFuture future = new CompletableFuture<>(); + Topic.PublishContext publishContext = new Topic.PublishContext() { + @Override + public void completed(Exception e, long ledgerId, long entryId) { + ReplicatedSubscriptionsController.this.completed(e, ledgerId, entryId); + if (e != null) { + future.completeExceptionally(e); + } else { + future.complete(PositionFactory.create(ledgerId, entryId)); + } + } + + @Override + public boolean isMarkerMessage() { + return true; + } + }; + + try { + topic.publishMessage(marker, publishContext); + } catch (Exception e) { + publishContext.completed(e, -1, -1); + } finally { + marker.release(); + } + return future; + } + /** * From Topic.PublishContext. */ diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsSnapshotBuilder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsSnapshotBuilder.java index e08b549f8aec9..df66794d751f8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsSnapshotBuilder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsSnapshotBuilder.java @@ -47,6 +47,7 @@ public class ReplicatedSubscriptionsSnapshotBuilder { private final boolean needTwoRounds; private boolean firstRoundComplete; + private boolean finalSnapshotMarkerPending; private long startTimeMillis; private final long timeoutMillis; @@ -116,15 +117,33 @@ synchronized void receivedSnapshotResponse(Position position, ReplicatedSubscrip return; } + if (finalSnapshotMarkerPending) { + return; + } + + finalSnapshotMarkerPending = true; if (log.isDebugEnabled()) { log.debug("[{}] Snapshot is complete {}", controller.topic().getName(), snapshotId); } // Snapshot is now complete, store it in the local topic Position p = position; - controller.writeMarker( + controller.writeMarkerAndGetPosition( Markers.newReplicatedSubscriptionsSnapshot(snapshotId, controller.localCluster(), - p.getLedgerId(), p.getEntryId(), responses)); - controller.snapshotCompleted(snapshotId); + p.getLedgerId(), p.getEntryId(), responses)) + .whenComplete((ignored, exception) -> { + if (exception != null) { + synchronized (ReplicatedSubscriptionsSnapshotBuilder.this) { + finalSnapshotMarkerPending = false; + } + if (log.isDebugEnabled()) { + log.debug("[{}] Failed to publish completed snapshot marker {}", + controller.topic().getName(), snapshotId, exception); + } + return; + } + + controller.snapshotCompleted(snapshotId); + }); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsControllerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsControllerTest.java index 818091ade2d1c..9af9a08cc3c0a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsControllerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsControllerTest.java @@ -24,11 +24,14 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import io.netty.buffer.ByteBuf; import java.time.Clock; import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -37,6 +40,8 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import lombok.Cleanup; import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback; import org.apache.bookkeeper.mledger.ManagedLedger; @@ -47,6 +52,7 @@ import org.apache.pulsar.broker.service.BacklogQuotaManager; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.Replicator; +import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.stats.OpenTelemetryReplicatedSubscriptionStats; import org.apache.pulsar.common.api.proto.MarkerType; import org.apache.pulsar.common.policies.data.BacklogQuota; @@ -60,6 +66,140 @@ @Test(groups = "broker-replication") public class ReplicatedSubscriptionsControllerTest { + @Test + @SuppressWarnings("unchecked") + public void testFinalSnapshotMarkerPublishFailureKeepsSnapshotPending() { + PulsarService pulsar = mock(PulsarService.class); + ScheduledExecutorService executor = mock(ScheduledExecutorService.class); + @SuppressWarnings("rawtypes") + ScheduledFuture timer = mock(ScheduledFuture.class); + ServiceConfiguration config = new ServiceConfiguration(); + config.setReplicatedSubscriptionsSnapshotFrequencyMillis(60_000); + config.setReplicatedSubscriptionsSnapshotTimeoutSeconds(3); + OpenTelemetryReplicatedSubscriptionStats stats = mock(OpenTelemetryReplicatedSubscriptionStats.class); + BrokerService brokerService = mock(BrokerService.class); + PersistentTopic topic = mock(PersistentTopic.class); + Replicator replicator = mock(Replicator.class); + List publishContexts = new ArrayList<>(); + AtomicReference scheduledSnapshotTask = new AtomicReference<>(); + + when(topic.getName()).thenReturn("persistent://public/default/t1"); + when(topic.getBrokerService()).thenReturn(brokerService); + when(topic.getLastMaxReadPositionMovedForwardTimestamp()).thenReturn(1L); + when(topic.getReplicators()).thenReturn(Map.of("remote", replicator)); + when(replicator.isConnected()).thenReturn(true); + when(brokerService.pulsar()).thenReturn(pulsar); + when(pulsar.getExecutor()).thenReturn(executor); + when(pulsar.getConfiguration()).thenReturn(config); + when(pulsar.getOpenTelemetryReplicatedSubscriptionStats()).thenReturn(stats); + when(executor.scheduleAtFixedRate(any(Runnable.class), anyLong(), anyLong(), any(TimeUnit.class))) + .thenAnswer(invocation -> { + scheduledSnapshotTask.set(invocation.getArgument(0, Runnable.class)); + return timer; + }); + doAnswer(invocation -> { + publishContexts.add(invocation.getArgument(1, Topic.PublishContext.class)); + return null; + }).when(topic).publishMessage(any(ByteBuf.class), any(Topic.PublishContext.class)); + + ReplicatedSubscriptionsController controller = new ReplicatedSubscriptionsController(topic, "local"); + String snapshotId = null; + try { + Assert.assertNotNull(scheduledSnapshotTask.get()); + scheduledSnapshotTask.get().run(); + Assert.assertEquals(controller.pendingSnapshots().size(), 1); + snapshotId = controller.pendingSnapshots().keySet().iterator().next(); + Assert.assertEquals(publishContexts.size(), 1); + + ByteBuf responseMarker = Markers.newReplicatedSubscriptionsSnapshotResponse(snapshotId, "local", + "remote", 11, 11); + try { + Commands.skipMessageMetadata(responseMarker); + controller.receivedReplicatedSubscriptionMarker(PositionFactory.create(1, 1), + MarkerType.REPLICATED_SUBSCRIPTION_SNAPSHOT_RESPONSE_VALUE, responseMarker); + } finally { + responseMarker.release(); + } + Assert.assertEquals(publishContexts.size(), 2); + + publishContexts.get(1).completed(new RuntimeException("final marker publish failed"), -1, -1); + + Assert.assertTrue(controller.pendingSnapshots().containsKey(snapshotId), + "Snapshot should remain pending after the final snapshot marker publish fails"); + Assert.assertFalse(controller.getLastCompletedSnapshotId().isPresent(), + "Failed final snapshot marker publish should not update the last completed snapshot id"); + } finally { + if (snapshotId != null && controller.pendingSnapshots().containsKey(snapshotId)) { + controller.snapshotCompleted(snapshotId); + } + controller.close(); + } + } + + @Test + @SuppressWarnings("unchecked") + public void testTimeoutCleanupDoesNotRecordTimeoutWhenSnapshotCompletedConcurrently() { + PulsarService pulsar = mock(PulsarService.class); + ScheduledExecutorService executor = mock(ScheduledExecutorService.class); + @SuppressWarnings("rawtypes") + ScheduledFuture timer = mock(ScheduledFuture.class); + ServiceConfiguration config = new ServiceConfiguration(); + config.setReplicatedSubscriptionsSnapshotFrequencyMillis(60_000); + config.setReplicatedSubscriptionsSnapshotTimeoutSeconds(3); + OpenTelemetryReplicatedSubscriptionStats stats = mock(OpenTelemetryReplicatedSubscriptionStats.class); + BrokerService brokerService = mock(BrokerService.class); + PersistentTopic topic = mock(PersistentTopic.class); + Replicator replicator = mock(Replicator.class); + AtomicReference scheduledSnapshotTask = new AtomicReference<>(); + + when(topic.getName()).thenReturn("persistent://public/default/t1"); + when(topic.getBrokerService()).thenReturn(brokerService); + when(topic.getLastMaxReadPositionMovedForwardTimestamp()).thenReturn(1L, 1L, 0L, 0L); + when(topic.getReplicators()).thenReturn(Map.of("remote", replicator)); + when(replicator.isConnected()).thenReturn(true); + when(brokerService.pulsar()).thenReturn(pulsar); + when(pulsar.getExecutor()).thenReturn(executor); + when(pulsar.getConfiguration()).thenReturn(config); + when(pulsar.getOpenTelemetryReplicatedSubscriptionStats()).thenReturn(stats); + when(executor.scheduleAtFixedRate(any(Runnable.class), anyLong(), anyLong(), any(TimeUnit.class))) + .thenAnswer(invocation -> { + scheduledSnapshotTask.set(invocation.getArgument(0, Runnable.class)); + return timer; + }); + + ReplicatedSubscriptionsController controller = new ReplicatedSubscriptionsController(topic, "local"); + try { + Assert.assertNotNull(scheduledSnapshotTask.get()); + scheduledSnapshotTask.get().run(); + Assert.assertEquals(controller.pendingSnapshots().size(), 1); + String snapshotId = controller.pendingSnapshots().keySet().iterator().next(); + ReplicatedSubscriptionsSnapshotBuilder originalBuilder = controller.pendingSnapshots().get(snapshotId); + + AtomicBoolean completedDuringTimeoutCheck = new AtomicBoolean(); + ReplicatedSubscriptionsSnapshotBuilder completedBuilder = + new ReplicatedSubscriptionsSnapshotBuilder(controller, Set.of("remote"), config, + Clock.systemUTC()) { + @Override + boolean isTimedOut() { + if (completedDuringTimeoutCheck.compareAndSet(false, true)) { + controller.snapshotCompleted(snapshotId); + } + return true; + } + }; + Assert.assertTrue(controller.pendingSnapshots().replace(snapshotId, originalBuilder, completedBuilder)); + + scheduledSnapshotTask.get().run(); + + Assert.assertTrue(completedDuringTimeoutCheck.get()); + Assert.assertFalse(controller.pendingSnapshots().containsKey(snapshotId)); + verify(stats).recordSnapshotCompleted(anyLong()); + verify(stats, never()).recordSnapshotTimedOut(anyLong()); + } finally { + controller.close(); + } + } + @Test public void testSnapshotRequestWhenReplicatorRemovedConcurrentlyDoesNotThrow() throws Exception { // Use a real PersistentTopic instance so that the replicator removal happens via the production code path diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsSnapshotBuilderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsSnapshotBuilderTest.java index 69c11edb3469d..400046136f5d4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsSnapshotBuilderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsSnapshotBuilderTest.java @@ -31,6 +31,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Set; +import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.common.api.proto.ReplicatedSubscriptionsSnapshot; @@ -74,6 +75,14 @@ public void setup() { }) .when(controller) .writeMarker(any(ByteBuf.class)); + doAnswer(invocation -> { + ByteBuf marker = invocation.getArgument(0, ByteBuf.class); + Commands.skipMessageMetadata(marker); + markers.add(marker); + return CompletableFuture.completedFuture(PositionFactory.create(1, 1)); + }) + .when(controller) + .writeMarkerAndGetPosition(any(ByteBuf.class)); } @AfterMethod From c27f9bd60ebbcc5c13006903926e2847821c1f44 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Sat, 25 Jul 2026 19:54:45 +0300 Subject: [PATCH 142/213] [fix][build][branch-4.0] Fix spotbugs failure in OffloadPoliciesImpl --- .../pulsar/common/policies/data/OffloadPoliciesImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java index 14b08e05caf3c..af47712b98402 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java @@ -70,8 +70,8 @@ public class OffloadPoliciesImpl implements Serializable, OffloadPolicies { public static final String DRIVER_FILESYSTEM = "filesystem"; public static final String DRIVER_AZUREBLOB = "azureblob"; public static final String DRIVER_ALIYUN_OSS = "aliyun-oss"; - public static final List INTERNAL_SUPPORTED_DRIVER = Arrays.asList(DRIVER_S3, - DRIVER_AWS_S3, DRIVER_GOOGLE_CLOUD_STORAGE, DRIVER_FILESYSTEM, DRIVER_AZUREBLOB, DRIVER_ALIYUN_OSS); + public static final List INTERNAL_SUPPORTED_DRIVER = Collections.unmodifiableList(Arrays.asList(DRIVER_S3, + DRIVER_AWS_S3, DRIVER_GOOGLE_CLOUD_STORAGE, DRIVER_FILESYSTEM, DRIVER_AZUREBLOB, DRIVER_ALIYUN_OSS)); public static final List DRIVER_NAMES; static { String extraDrivers = System.getProperty("pulsar.extra.offload.drivers", ""); From 2dc3fafe2a093d8249f2aee093f9df001311f5fd Mon Sep 17 00:00:00 2001 From: void-ptr974 Date: Sun, 26 Jul 2026 00:52:47 +0800 Subject: [PATCH 143/213] [fix][broker] Fix incorrect listener URLs returned by ModularLoadManager lookups (#26245) (cherry picked from commit 931f9fb0d8aa1fe15446e5ebb1e867868e87fe2c) --- .../broker/namespace/NamespaceService.java | 29 +++- .../NamespaceServiceLookupOptionsTest.java | 163 ++++++++++++++++++ 2 files changed, 186 insertions(+), 6 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceLookupOptionsTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 8baa9bef67932..f37f945d42566 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -455,11 +455,27 @@ public boolean registerNamespace(NamespaceName nsname, boolean ensureOwned) thro } } - private final Map>> + // The two maps keep authoritative and non-authoritative lookups separate. The key contains every + // remaining option that affects the lookup result or its side effects. + private final Map>> findingBundlesAuthoritative = new ConcurrentHashMap<>(); - private final Map>> + private final Map>> findingBundlesNotAuthoritative = new ConcurrentHashMap<>(); + /** + * Key for coalescing lookup requests handled by this lookup path. + * + *

Lookup properties and the requestHttps flag are intentionally excluded because this lookup path + * does not consume them. + */ + private record LookupRequestKey(NamespaceBundle bundle, boolean readOnly, boolean loadTopicsInBundle, + String advertisedListenerName) { + private static LookupRequestKey from(NamespaceBundle bundle, LookupOptions options) { + return new LookupRequestKey(bundle, options.isReadOnly(), options.isLoadTopicsInBundle(), + options.hasAdvertisedListenerName() ? options.getAdvertisedListenerName() : null); + } + } + /** * Main internal method to lookup and setup ownership of service unit to a broker. * @@ -473,18 +489,19 @@ private CompletableFuture> findBrokerServiceUrl( LOG.debug("findBrokerServiceUrl: {} - options: {}", bundle, options); } - Map>> targetMap; + Map>> targetMap; if (options.isAuthoritative()) { targetMap = findingBundlesAuthoritative; } else { targetMap = findingBundlesNotAuthoritative; } + LookupRequestKey lookupRequestKey = LookupRequestKey.from(bundle, options); - return targetMap.computeIfAbsent(bundle, (k) -> { + return targetMap.computeIfAbsent(lookupRequestKey, (k) -> { CompletableFuture> future = new CompletableFuture<>(); // First check if we or someone else already owns the bundle - ownershipCache.getOwnerAsync(bundle).thenAccept(nsData -> { + getOwnershipCache().getOwnerAsync(bundle).thenAccept(nsData -> { if (nsData.isEmpty()) { // No one owns this bundle @@ -528,7 +545,7 @@ private CompletableFuture> findBrokerServiceUrl( }); future.whenComplete((r, t) -> pulsar.getExecutor().execute( - () -> targetMap.remove(bundle) + () -> targetMap.remove(lookupRequestKey, future) )); return future; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceLookupOptionsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceLookupOptionsTest.java new file mode 100644 index 0000000000000..13fe6a4d20618 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceLookupOptionsTest.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.namespace; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import java.net.URI; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.pulsar.broker.loadbalance.extensions.manager.RedirectManager; +import org.apache.pulsar.broker.lookup.LookupResult; +import org.apache.pulsar.broker.service.BrokerTestBase; +import org.apache.pulsar.common.naming.NamespaceBundle; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class NamespaceServiceLookupOptionsTest extends BrokerTestBase { + + @BeforeMethod + @Override + protected void setup() throws Exception { + super.baseSetup(); + } + + @AfterMethod(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @DataProvider(name = "differentLookupOptions") + public Object[][] differentLookupOptions() { + return new Object[][] { + { + LookupOptions.builder().advertisedListenerName("listener-a").build(), + LookupOptions.builder().advertisedListenerName("listener-b").build() + }, + { + LookupOptions.builder().readOnly(true).build(), + LookupOptions.builder().readOnly(false).build() + }, + { + LookupOptions.builder().loadTopicsInBundle(true).build(), + LookupOptions.builder().loadTopicsInBundle(false).build() + } + }; + } + + @Test(dataProvider = "differentLookupOptions") + public void testLookupRequestsWithDifferentOptionsAreNotCoalesced( + LookupOptions firstOptions, LookupOptions secondOptions) throws Exception { + TopicName topic = TopicName.get("persistent://public/default/lookup-options-" + UUID.randomUUID()); + NamespaceBundle bundle = pulsar.getNamespaceService().getBundle(topic); + NamespaceService namespaceService = pulsar.getNamespaceService(); + OwnershipCache ownershipCache = mock(OwnershipCache.class); + CompletableFuture> ownerFuture = new CompletableFuture<>(); + doReturn(CompletableFuture.completedFuture(bundle)).when(namespaceService).getBundleAsync(any()); + doReturn(ownershipCache).when(namespaceService).getOwnershipCache(); + when(ownershipCache.getOwnerAsync(bundle)).thenReturn(ownerFuture); + stubRedirectManager(namespaceService); + + CompletableFuture> firstLookup = + namespaceService.getBrokerServiceUrlAsync(topic, firstOptions); + CompletableFuture> secondLookup = + namespaceService.getBrokerServiceUrlAsync(topic, secondOptions); + + verify(ownershipCache, times(2)).getOwnerAsync(bundle); + ownerFuture.complete(Optional.of(lookupOwnerWithAdvertisedListeners())); + + assertLookupResultUsesOptions(firstLookup.get(5, TimeUnit.SECONDS).orElseThrow(), firstOptions); + assertLookupResultUsesOptions(secondLookup.get(5, TimeUnit.SECONDS).orElseThrow(), secondOptions); + } + + @Test + public void testEquivalentLookupRequestsAreCoalesced() throws Exception { + TopicName topic = TopicName.get("persistent://public/default/lookup-options-" + UUID.randomUUID()); + NamespaceBundle bundle = pulsar.getNamespaceService().getBundle(topic); + NamespaceService namespaceService = pulsar.getNamespaceService(); + OwnershipCache ownershipCache = mock(OwnershipCache.class); + CompletableFuture> ownerFuture = new CompletableFuture<>(); + LookupOptions options = LookupOptions.builder().advertisedListenerName("listener-a").build(); + doReturn(CompletableFuture.completedFuture(bundle)).when(namespaceService).getBundleAsync(any()); + doReturn(ownershipCache).when(namespaceService).getOwnershipCache(); + when(ownershipCache.getOwnerAsync(bundle)).thenReturn(ownerFuture); + stubRedirectManager(namespaceService); + + CompletableFuture> firstLookup = + namespaceService.getBrokerServiceUrlAsync(topic, options); + CompletableFuture> secondLookup = namespaceService.getBrokerServiceUrlAsync(topic, + LookupOptions.builder().advertisedListenerName("listener-a").build()); + + verify(ownershipCache).getOwnerAsync(bundle); + ownerFuture.complete(Optional.of(lookupOwnerWithAdvertisedListeners())); + + assertLookupResultUsesOptions(firstLookup.get(5, TimeUnit.SECONDS).orElseThrow(), options); + assertLookupResultUsesOptions(secondLookup.get(5, TimeUnit.SECONDS).orElseThrow(), options); + } + + /** + * Replaces the redirect manager with one that answers immediately, so that the lookup reaches the + * request coalescing logic on the calling thread. Otherwise the redirect check performs a metadata + * store read and the lookups would be handled asynchronously. + */ + private static void stubRedirectManager(NamespaceService namespaceService) throws Exception { + RedirectManager redirectManager = mock(RedirectManager.class); + when(redirectManager.findRedirectLookupResultAsync()) + .thenReturn(CompletableFuture.completedFuture(Optional.empty())); + FieldUtils.writeDeclaredField(namespaceService, "redirectManager", redirectManager, true); + } + + private static NamespaceEphemeralData lookupOwnerWithAdvertisedListeners() { + Map advertisedListeners = Map.of( + "listener-a", AdvertisedListener.builder() + .brokerServiceUrl(URI.create("pulsar://listener-a:6650")) + .brokerHttpUrl(URI.create("http://listener-a:8080")) + .build(), + "listener-b", AdvertisedListener.builder() + .brokerServiceUrl(URI.create("pulsar://listener-b:6650")) + .brokerHttpUrl(URI.create("http://listener-b:8080")) + .build()); + return new NamespaceEphemeralData("pulsar://broker-1:6650", null, + "http://broker-1:8080", null, false, advertisedListeners); + } + + private static void assertLookupResultUsesOptions(LookupResult lookupResult, LookupOptions options) { + if (options.hasAdvertisedListenerName()) { + assertEquals(lookupResult.getLookupData().getBrokerUrl(), + "pulsar://" + options.getAdvertisedListenerName() + ":6650"); + } else { + assertEquals(lookupResult.getLookupData().getBrokerUrl(), "pulsar://broker-1:6650"); + } + } +} From f9a2f1f90779f0f67dbadbbfdd2056be1cdaa740 Mon Sep 17 00:00:00 2001 From: void-ptr974 Date: Sun, 26 Jul 2026 01:45:43 +0800 Subject: [PATCH 144/213] [fix][broker] Fix delayed message index data loss when trimming overlapping bucket snapshots (#26240) (cherry picked from commit c5cb6fc27d40010768c9218a951901dc679f60fc) --- .../bucket/BucketDelayedDeliveryTracker.java | 10 ++++- .../BucketDelayedDeliveryTrackerTest.java | 42 +++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index 37ee442b85a55..68ce805c0add2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -850,8 +850,14 @@ private synchronized CompletableFuture asyncTrimImmutableBuckets() { } ManagedLedger ledger = dispatcher.getCursor().getManagedLedger(); - Map, ImmutableBucket> toBeDeletedBuckets = - new HashMap<>(immutableBuckets.subRangeMap(Range.lessThan(firstLedgerId)).asMapOfRanges()); + Map, ImmutableBucket> toBeDeletedBuckets = new HashMap<>(); + // subRangeMap returns clipped intersection ranges. Snapshot deletion must use the original + // bucket range, so only select buckets whose complete range precedes the first live ledger. + immutableBuckets.asMapOfRanges().forEach((range, bucket) -> { + if (range.upperEndpoint() < firstLedgerId) { + toBeDeletedBuckets.put(range, bucket); + } + }); if (toBeDeletedBuckets.isEmpty()) { return CompletableFuture.completedFuture(null); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java index 4a73f335cf485..14a257f303d0d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java @@ -546,6 +546,16 @@ public CompletableFuture deleteBucketSnapshot(long bucketId) { } } + private static class RecordingDeleteStorage extends MockBucketSnapshotStorage { + final AtomicLong deleteCalls = new AtomicLong(); + + @Override + public CompletableFuture deleteBucketSnapshot(long bucketId) { + deleteCalls.incrementAndGet(); + return super.deleteBucketSnapshot(bucketId); + } + } + private TrackerWithStorage createTrackerWithMockLedger(long firstLedgerId, int maxNumBuckets) throws Exception { return createTrackerWithMockLedger(firstLedgerId, maxNumBuckets, new MockBucketSnapshotStorage()); @@ -618,6 +628,38 @@ public void testTrimRemovesOrphanedBuckets() throws Exception { ts.close(); } + @Test + public void testTrimDoesNotDeleteBucketOverlappingFirstActiveLedger() throws Exception { + RecordingDeleteStorage storage = new RecordingDeleteStorage(); + TrackerWithStorage ts = createTrackerWithMockLedger(3L, 4, storage); + try { + for (long ledgerId = 1; ledgerId <= 6; ledgerId++) { + ts.tracker.addMessage(ledgerId, 0L, 1000L); + } + + Awaitility.await().untilAsserted(() -> { + assertEquals(ts.tracker.getImmutableBuckets().asMapOfRanges().size(), 1); + ImmutableBucket bucket = ts.tracker.getImmutableBuckets().asMapOfRanges() + .values().iterator().next(); + assertTrue(bucket.getSnapshotCreateFuture().orElseThrow().isDone()); + }); + + // The fifth immutable bucket triggers trimming. All buckets have one snapshot segment, + // so merging does not remove any snapshots during this test. + for (long ledgerId = 7; ledgerId <= 26; ledgerId++) { + ts.tracker.addMessage(ledgerId, 0L, 1000L); + } + Awaitility.await().untilAsserted( + () -> assertEquals(ts.tracker.getImmutableBuckets().asMapOfRanges().size(), 5)); + + Awaitility.await().pollDelay(1, TimeUnit.SECONDS).atMost(2, TimeUnit.SECONDS).untilAsserted( + () -> assertEquals(storage.deleteCalls.get(), 0L, + "A bucket overlapping the first active ledger must not be deleted")); + } finally { + ts.close(); + } + } + @Test public void testTrimHandlesDeleteFailure() throws Exception { long firstLedgerId = 50L; From be5ddc332eb7aa5799caa24d1cca122d75ae9cad Mon Sep 17 00:00:00 2001 From: SongOf <46475785+SongOf@users.noreply.github.com> Date: Sun, 26 Jul 2026 01:49:54 +0800 Subject: [PATCH 145/213] [fix][broker] Fix silently dropped acknowledgement failures in PulsarMetadataEventSynchronizer (#26237) Co-authored-by: maxlisongsong (cherry picked from commit c6fd26f123d85bd771b236f853f0931313fecb20) --- .../PulsarMetadataEventSynchronizer.java | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarMetadataEventSynchronizer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarMetadataEventSynchronizer.java index a5fac333ae57e..25af3597c2006 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarMetadataEventSynchronizer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarMetadataEventSynchronizer.java @@ -34,6 +34,7 @@ import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.ConsumerBuilder; +import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.Schema; @@ -197,25 +198,16 @@ private void startConsumer() { listeners.size()); try { if (listeners.size() == 0) { - c.acknowledgeAsync(msg); + acknowledgeAfter(CompletableFuture.completedFuture(null), c, msg); return; } if (listeners.size() == 1) { - listeners.get(0).apply(msg.getValue()).thenApply(__ -> c.acknowledgeAsync(msg)) - .exceptionally(ex -> { - log.warn("Failed to synchronize {} for {}", msg.getMessageId(), topicName, - ex.getCause()); - return null; - }); + acknowledgeAfter(listeners.get(0).apply(msg.getValue()), c, msg); } else { - FutureUtil - .waitForAll(listeners.stream().map(listener -> listener.apply(msg.getValue())) - .collect(Collectors.toList())) - .thenApply(__ -> c.acknowledgeAsync(msg)).exceptionally(ex -> { - log.warn("Failed to synchronize {} for {}", msg.getMessageId(), topicName); - return null; - }); + acknowledgeAfter( + FutureUtil.waitForAll(listeners.stream().map(listener -> listener.apply(msg.getValue())) + .collect(Collectors.toList())), c, msg); } } catch (Exception e) { log.warn("Failed to synchronize {} for {}", msg.getMessageId(), topicName); @@ -247,6 +239,19 @@ private void startConsumer() { }); } + /** + * Acknowledge {@code msg} only after {@code processed} completes, and log (rather than silently drop) + * an exception from either the processing stage or the acknowledgement itself. + */ + private CompletableFuture acknowledgeAfter(CompletableFuture processed, Consumer c, + Message msg) { + return processed.thenCompose(__ -> c.acknowledgeAsync(msg)) + .exceptionally(ex -> { + log.warn("Failed to synchronize {} for {}", msg.getMessageId(), topicName, ex); + return null; + }); + } + public boolean isStarted() { return this.state == State.Started; } From 694a0566ba702b70a9d8af8a4859b243c08e8f5c Mon Sep 17 00:00:00 2001 From: sbourkeostk <58816076+sbourkeostk@users.noreply.github.com> Date: Sat, 25 Jul 2026 18:50:52 +0100 Subject: [PATCH 146/213] [fix][fn] Forward source message properties in Python runtime (#26191) Co-authored-by: Stephen Bourke (cherry picked from commit f07e0ed5899d29351774df25b138d4b92557ae35) --- .../src/main/python/python_instance.py | 5 +- .../instance/src/main/python/util.py | 4 +- .../src/scripts/run_python_instance_tests.sh | 4 +- .../src/test/python/test_python_instance.py | 61 ++++++++++++++++++- 4 files changed, 68 insertions(+), 6 deletions(-) diff --git a/pulsar-functions/instance/src/main/python/python_instance.py b/pulsar-functions/instance/src/main/python/python_instance.py index 2ab3ccc46171c..5c57dfef79008 100755 --- a/pulsar-functions/instance/src/main/python/python_instance.py +++ b/pulsar-functions/instance/src/main/python/python_instance.py @@ -337,7 +337,10 @@ def process_result(self, output, msg): output_object = self.output_serde.serialize(output) if output_object is not None: - props = {"__pfn_input_topic__" : str(msg.topic), "__pfn_input_msg_id__" : base64ify(msg.message.message_id().serialize())} + props = {} + if self.instance_config.function_details.sink.forwardSourceMessageProperty: + props = msg.message.properties() + props.update({"__pfn_input_topic__" : str(msg.topic), "__pfn_input_msg_id__" : base64ify(msg.message.message_id().serialize())}) if self.effectively_once: self.producer.send_async(output_object, partial(self.done_producing, msg.consumer, msg.message, self.producer.topic()), diff --git a/pulsar-functions/instance/src/main/python/util.py b/pulsar-functions/instance/src/main/python/util.py index f5868093faea4..7b2a2d7b4d172 100755 --- a/pulsar-functions/instance/src/main/python/util.py +++ b/pulsar-functions/instance/src/main/python/util.py @@ -105,8 +105,8 @@ def __init__(self, t, hFunction, name="timer-thread"): self.t = t self.hFunction = hFunction self.thread = Timer(self.t, self.handle_function) - self.thread.setName(name) - self.thread.setDaemon(True) + self.thread.name = name + self.thread.daemon = True def handle_function(self): self.hFunction() diff --git a/pulsar-functions/instance/src/scripts/run_python_instance_tests.sh b/pulsar-functions/instance/src/scripts/run_python_instance_tests.sh index 5cb729686790f..11cec15e14c09 100755 --- a/pulsar-functions/instance/src/scripts/run_python_instance_tests.sh +++ b/pulsar-functions/instance/src/scripts/run_python_instance_tests.sh @@ -21,11 +21,11 @@ # Make sure dependencies are installed pip3 install mock --user -pip3 install protobuf==3.20.1 --user +pip3 install protobuf==6.31.1 --user pip3 install fastavro --user CUR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" PULSAR_HOME="$( cd "$CUR_DIR/../../../../" >/dev/null && pwd )" # run instance tests -PULSAR_HOME=${PULSAR_HOME} PYTHONPATH=${PULSAR_HOME}/pulsar-functions/instance/target/python-instance python3 -m unittest discover -v ${PULSAR_HOME}/pulsar-functions/instance/target/python-instance/tests +PULSAR_HOME=${PULSAR_HOME} PYTHONPATH=${PULSAR_HOME}/pulsar-functions/instance/src/main/python python3 -m unittest discover -v -s ${PULSAR_HOME}/pulsar-functions/instance/src/test/python diff --git a/pulsar-functions/instance/src/test/python/test_python_instance.py b/pulsar-functions/instance/src/test/python/test_python_instance.py index 975ddfd90288d..1e72db8545816 100644 --- a/pulsar-functions/instance/src/test/python/test_python_instance.py +++ b/pulsar-functions/instance/src/test/python/test_python_instance.py @@ -23,6 +23,12 @@ from mock import Mock import sys sys.modules['prometheus_client'] = Mock() +sys.modules['bookkeeper'] = Mock() +sys.modules['bookkeeper.types'] = Mock() +sys.modules['bookkeeper.common'] = Mock() +sys.modules['bookkeeper.common.exceptions'] = Mock() +sys.modules['bookkeeper.proto'] = Mock() +sys.modules['bookkeeper.proto.stream_pb2'] = Mock() from contextimpl import ContextImpl from python_instance import PythonInstance, InstanceConfig @@ -42,7 +48,8 @@ def __eq__(self, other): return Any() def setUp(self): - log.init_logger("INFO", "foo", os.environ.get("PULSAR_HOME") + "/conf/functions-logging/console_logging_config.ini") + if not hasattr(sys.stdout, 'logger'): + log.init_logger("INFO", "foo", os.environ.get("PULSAR_HOME") + "/conf/functions-logging/console_logging_config.ini") def test_context_publish(self): instance_id = 'test_instance_id' @@ -90,3 +97,55 @@ def test_context_ack_partitionedtopic(self): args, kwargs = consumer.acknowledge.call_args self.assertEqual(args[0], "test_message_id") + +class TestPropertiesForwarding(unittest.TestCase): + + def _setup_mock_instance(self, forward_property): + function_details = Function_pb2.FunctionDetails() + function_details.sink.topic = "test_sink_topic" + function_details.sink.forwardSourceMessageProperty = forward_property + + mock_pulsar_client = Mock() + mock_producer = Mock() + mock_pulsar_client.create_producer.return_value = mock_producer + + instance = PythonInstance('test_instance', 'test_func', '1.0', function_details, 100, 30, 'user_code', mock_pulsar_client, Mock(), 'test_cluster', 'test_url', None) + instance.producer = mock_producer + instance.contextimpl = Mock() + instance.contextimpl.get_message_partition_index.return_value = None + instance.output_schema = "DEFAULT_SCHEMA" + instance.output_serde = Mock() + instance.output_serde.serialize.return_value = b'serialized_output' + instance.effectively_once = False + + return instance, mock_producer + + def test_forwards_properties(self): + instance, mock_producer = self._setup_mock_instance(forward_property=True) + + mock_msg = Mock() + mock_msg.topic = "source-topic" + mock_msg.message.message_id().serialize.return_value = b'msg-id' + mock_msg.message.properties.return_value = {"custom-key": "custom-value"} + + instance.process_result("output-data", mock_msg) + + args, kwargs = mock_producer.send_async.call_args + self.assertIn("custom-key", kwargs['properties']) + self.assertEqual(kwargs['properties']["custom-key"], "custom-value") + self.assertIn("__pfn_input_topic__", kwargs['properties']) + + def test_do_not_forward_properties(self): + instance, mock_producer = self._setup_mock_instance(forward_property=False) + + mock_msg = Mock() + mock_msg.topic = "source-topic" + mock_msg.message.message_id().serialize.return_value = b'msg-id' + mock_msg.message.properties.return_value = {"custom-key": "custom-value"} + + instance.process_result("output-data", mock_msg) + + args, kwargs = mock_producer.send_async.call_args + self.assertNotIn("custom-key", kwargs['properties']) + self.assertIn("__pfn_input_topic__", kwargs['properties']) + From 1e80545e9d1d42aea89db07e5c9690474acd86c6 Mon Sep 17 00:00:00 2001 From: Dream95 <864197662@qq.com> Date: Sun, 26 Jul 2026 02:06:33 +0800 Subject: [PATCH 147/213] [fix][client] Fix unAckedMessageTracker cleanup on multi-topics batch ack (#26001) Signed-off-by: Dream95 (cherry picked from commit 3ebf796497bfda55c84ff985f54f0f25ac829734) --- .../client/impl/MultiTopicsConsumerImpl.java | 2 +- .../impl/MultiTopicsConsumerImplTest.java | 35 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index b3bc3f6a46f10..ee87c124545c7 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -529,7 +529,7 @@ protected CompletableFuture doAcknowledge(List messageIdList, } consumerToMessageIds.forEach((consumer, messageIds) -> { resultFutures.add(consumer.doAcknowledgeWithTxn(messageIds, ackType, properties, txn) - .thenAccept((res) -> messageIdList.forEach(unAckedMessageTracker::remove))); + .thenAccept((res) -> messageIds.forEach(unAckedMessageTracker::remove))); }); } return CompletableFuture.allOf(resultFutures.toArray(new CompletableFuture[0])); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java index b4d007d855f88..f16702577a30b 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImplTest.java @@ -23,6 +23,7 @@ import static org.apache.pulsar.client.impl.ClientTestFixtures.createPulsarClientMockWithMockedClientCnx; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -317,4 +318,38 @@ public void testOnTopicsExtendedRemovedTopicCleansUnackedMessages() { verify(partitionConsumer1).closeAsync(); } + @Test + @SuppressWarnings("unchecked") + public void testBatchAcknowledgeRemovesOnlyAckedMessageIdsFromTracker() { + String topic0 = "persistent://public/default/topic-a-partition-0"; + String topic1 = "persistent://public/default/topic-b-partition-0"; + + ConsumerConfigurationData conf = new ConsumerConfigurationData<>(); + conf.setSubscriptionName("subscriptionName"); + conf.setAckTimeoutMillis(1000); + MultiTopicsConsumerImpl impl = createMultiTopicsConsumer(conf); + impl.setState(HandlerState.State.Ready); + + ConsumerImpl consumer0 = mock(ConsumerImpl.class); + ConsumerImpl consumer1 = mock(ConsumerImpl.class); + CompletableFuture pendingAck = new CompletableFuture<>(); + when(consumer0.getTopic()).thenReturn(topic0); + when(consumer1.getTopic()).thenReturn(topic1); + when(consumer0.doAcknowledgeWithTxn(anyList(), any(), any(), any())).thenReturn(pendingAck); + when(consumer1.doAcknowledgeWithTxn(anyList(), any(), any(), any())).thenReturn(new CompletableFuture<>()); + + impl.consumers.put(topic0, consumer0); + impl.consumers.put(topic1, consumer1); + + TopicMessageIdImpl messageId0 = new TopicMessageIdImpl(topic0, new MessageIdImpl(1, 1, 0)); + TopicMessageIdImpl messageId1 = new TopicMessageIdImpl(topic1, new MessageIdImpl(2, 2, 0)); + impl.getUnAckedMessageTracker().add(messageId0); + impl.getUnAckedMessageTracker().add(messageId1); + + impl.acknowledgeAsync(Arrays.asList(messageId0, messageId1)); + pendingAck.complete(null); + + assertEquals(impl.getUnAckedMessageTracker().size(), 1); + } + } From 7cc56bb74db7325da270bfa5f45f48db7506c279 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Sat, 25 Jul 2026 22:54:39 +0300 Subject: [PATCH 148/213] [fix][broker] Fix TableViewLoadDataStoreImpl close deadlock that stalls broker shutdown (#26243) (cherry picked from commit aa3ec21b10fd34f681b7c259e0070828afdc665c) --- .../store/TableViewLoadDataStoreImpl.java | 21 +++++++- .../extensions/store/LoadDataStoreTest.java | 53 +++++++++++++++++++ 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java index 3ce44a1e65a73..21b687337c9a7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.store; +import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.util.Map; import java.util.Optional; @@ -145,7 +146,14 @@ public synchronized void init() throws IOException { public synchronized void closeTableView() throws IOException { validateState(); if (tableView != null) { - tableView.close(); + // Close asynchronously without waiting: the close future may only be able to complete on the + // client's internal executor thread, and that thread can be blocked on this store's monitor + // (e.g. a ServiceUnitStateChannel StateChangeListener calling pushAsync/removeAsync), so a + // blocking close while holding the monitor could deadlock. + tableView.closeAsync().exceptionally(e -> { + log.warn("Failed to close table view on {}", topic, e); + return null; + }); tableView = null; } } @@ -160,7 +168,11 @@ public synchronized void start() throws LoadDataStoreException { private synchronized void closeProducer() throws IOException { validateState(); if (producer != null) { - producer.close(); + // Close asynchronously without waiting; see closeTableView() for the deadlock rationale. + producer.closeAsync().exceptionally(e -> { + log.warn("Failed to close producer on {}", topic, e); + return null; + }); producer = null; } } @@ -210,6 +222,11 @@ public synchronized void shutdown() throws IOException { isShutdown = true; } + @VisibleForTesting + synchronized void setTableView(TableView tableView) { + this.tableView = tableView; + } + private String validateProducer() { if (isShutdown) { return SHUTDOWN_ERR_MSG; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java index fe03446e58b7d..d456369b1e768 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java @@ -18,9 +18,12 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.store; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertThrows; @@ -29,12 +32,15 @@ import java.util.HashMap; import java.util.Map; import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; import lombok.AllArgsConstructor; import lombok.Cleanup; import lombok.Data; import lombok.NoArgsConstructor; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.client.api.TableView; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.policies.data.TenantInfoImpl; @@ -187,6 +193,53 @@ public void testProducerStop() throws Exception { loadDataStore.removeAsync("2").get(); } + @Test(timeOut = 30_000) + @SuppressWarnings("unchecked") + public void testShutdownDoesNotDeadlockWithConcurrentStoreAccess() throws Exception { + String topic = TopicDomain.persistent + "://" + NamespaceName.SYSTEM_NAMESPACE + "/" + UUID.randomUUID(); + var loadDataStore = + (TableViewLoadDataStoreImpl) LoadDataStoreFactory.create(pulsar, topic, Integer.class); + loadDataStore.start(); + loadDataStore.closeTableView(); + + // Replace the table view with a stub whose close future completes only after another thread has + // called a synchronized method of the store. This mirrors the production interleaving where the + // reader close future completes on the client's internal executor thread while a channel + // StateChangeListener on that same thread is blocked in removeAsync() waiting for the store + // monitor held by the closing thread. + CompletableFuture closeSignal = new CompletableFuture<>(); + CountDownLatch closeStarted = new CountDownLatch(1); + TableView stubTableView = mock(TableView.class); + when(stubTableView.closeAsync()).thenAnswer(invocation -> { + closeStarted.countDown(); + return closeSignal; + }); + doAnswer(invocation -> { + closeStarted.countDown(); + closeSignal.get(); + return null; + }).when(stubTableView).close(); + loadDataStore.setTableView(stubTableView); + + Thread concurrentAccess = new Thread(() -> { + try { + closeStarted.await(); + loadDataStore.removeAsync("key"); + closeSignal.complete(null); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + concurrentAccess.start(); + try { + loadDataStore.shutdown(); + } finally { + concurrentAccess.join(5000); + } + assertFalse(concurrentAccess.isAlive()); + assertTrue(closeSignal.isDone()); + } + @Test public void testShutdown() throws Exception { String topic = TopicDomain.persistent + "://" + NamespaceName.SYSTEM_NAMESPACE + "/" + UUID.randomUUID(); From 65c0dba9dbfb8f1d40af30d1f24a71ec0a8d9ef2 Mon Sep 17 00:00:00 2001 From: void-ptr974 Date: Sun, 26 Jul 2026 04:20:27 +0800 Subject: [PATCH 149/213] [fix][broker] Prevent stale service unit callbacks from dropping active lookup and cleanup jobs (#26146) (cherry picked from commit 32146ae350ca0526b04c3d089e2e4d2cde648688) --- .../channel/ServiceUnitStateChannelImpl.java | 37 +++-- .../channel/ServiceUnitStateChannelTest.java | 156 +++++++++++++++++- 2 files changed, 181 insertions(+), 12 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index a0e9f1f73e251..731a2196d2e07 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -232,6 +232,16 @@ public ServiceUnitStateChannelImpl(PulsarService pulsar) { this.channelState = Constructed; } + @VisibleForTesting + Map> getOwnerRequests() { + return getOwnerRequests; + } + + @VisibleForTesting + Map> getCleanupJobs() { + return cleanupJobs; + } + @Override public void scheduleOwnershipMonitor() { if (monitorTask == null) { @@ -846,13 +856,15 @@ brokerId, getLogEventTag(data), serviceUnit, } } - private void handleSkippedEvent(String serviceUnit) { + @VisibleForTesting + void handleSkippedEvent(String serviceUnit) { var getOwnerRequest = getOwnerRequests.get(serviceUnit); if (getOwnerRequest != null) { var data = tableview.get(serviceUnit); if (data != null && data.state() == Owned) { getOwnerRequest.complete(data.dstBroker()); - getOwnerRequests.remove(serviceUnit); + // Completing the request can run callbacks that install a newer request for the same service unit. + getOwnerRequests.remove(serviceUnit, getOwnerRequest); stateChangeListeners.notify(serviceUnit, data, null); } } @@ -1015,7 +1027,8 @@ private CompletableFuture deferGetOwner(String serviceUnit) { return future; } - private CompletableFuture dedupeGetOwnerRequest(String serviceUnit) { + @VisibleForTesting + CompletableFuture dedupeGetOwnerRequest(String serviceUnit) { var requested = new MutableObject>(); try { @@ -1048,7 +1061,8 @@ private CompletableFuture dedupeGetOwnerRequest(String serviceUnit) { var future = requested.getValue(); if (future != null) { future.whenComplete((__, e) -> { - getOwnerRequests.remove(serviceUnit); + // The request may have been replaced by a later lookup before this callback runs. + getOwnerRequests.remove(serviceUnit, future); if (e != null) { log.warn("{} failed to getOwner for serviceUnit:{}", brokerId, serviceUnit, e); } @@ -1325,12 +1339,13 @@ private MetadataState getMetadataState() { private void handleBrokerCreationEvent(String broker) { - if (!cleanupJobs.isEmpty() && cleanupJobs.containsKey(broker)) { + CompletableFuture cleanupJob = cleanupJobs.get(broker); + if (cleanupJob != null) { healthCheckBrokerAsync(broker) .thenAccept(__ -> { - CompletableFuture future = cleanupJobs.remove(broker); - if (future != null) { - future.cancel(false); + // The health check is async; only cancel the cleanup job observed before the check. + if (cleanupJobs.remove(broker, cleanupJob)) { + cleanupJob.cancel(false); totalInactiveBrokerCleanupCancelledCnt++; log.info("Successfully cancelled the ownership cleanup for broker:{}." + " Active cleanup job count:{}", @@ -1391,7 +1406,8 @@ private boolean channelDisabled() { return false; } - private void scheduleCleanup(String broker, long delayInSecs) { + @VisibleForTesting + void scheduleCleanup(String broker, long delayInSecs) { var scheduled = new MutableObject>(); try { if (channelDisabled()) { @@ -1419,7 +1435,8 @@ private void scheduleCleanup(String broker, long delayInSecs) { var future = scheduled.getValue(); if (future != null) { future.whenComplete((v, ex) -> { - cleanupJobs.remove(broker); + // The job may have been replaced by a later cleanup schedule before this callback runs. + cleanupJobs.remove(broker, future); }); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java index 6381a2851a271..e5be773d7fda9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java @@ -95,6 +95,7 @@ import org.apache.pulsar.client.api.TableView; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.naming.TopicVersion; import org.apache.pulsar.common.policies.data.TopicType; import org.apache.pulsar.common.stats.Metrics; import org.apache.pulsar.common.util.FutureUtil; @@ -280,6 +281,104 @@ public void channelOwnerTest() throws Exception { } } + @Test(priority = 1) + public void testCompletedGetOwnerRequestDoesNotRemoveNewRequest() { + ServiceUnitStateChannelImpl channel = (ServiceUnitStateChannelImpl) channel1; + String serviceUnit = namespaceName + "/0x10000000_0x10000001"; + var getOwnerRequests = channel.getOwnerRequests(); + getOwnerRequests.remove(serviceUnit); + CompletableFuture oldRequest = channel.dedupeGetOwnerRequest(serviceUnit); + assertEquals(getOwnerRequests.get(serviceUnit), oldRequest); + + CompletableFuture newRequest = null; + try { + // State-event handlers remove the current request before completing it, allowing a later lookup + // to install a new request generation before the old request's completion cleanup runs. + assertTrue(getOwnerRequests.remove(serviceUnit, oldRequest)); + newRequest = channel.dedupeGetOwnerRequest(serviceUnit); + assertTrue(newRequest != oldRequest); + assertTrue(getOwnerRequests.get(serviceUnit) == newRequest); + + // The previous unconditional removal would remove newRequest here. + assertTrue(oldRequest.complete(brokerId1)); + + assertTrue(getOwnerRequests.get(serviceUnit) == newRequest, + "A stale get-owner cleanup must not remove a newer request future"); + + assertTrue(newRequest.complete(brokerId2)); + assertFalse(getOwnerRequests.containsKey(serviceUnit), + "The newer request must remove itself after completion"); + } finally { + getOwnerRequests.remove(serviceUnit); + oldRequest.cancel(false); + if (newRequest != null) { + newRequest.cancel(false); + } + } + } + + @Test(priority = 1) + public void testSkippedEventDoesNotRemoveNewGetOwnerRequest() throws Exception { + ServiceUnitStateChannelImpl channel = (ServiceUnitStateChannelImpl) channel1; + String serviceUnit = namespaceName + "/0x10000002_0x10000003"; + var getOwnerRequests = channel.getOwnerRequests(); + CompletableFuture oldRequest = new CompletableFuture<>(); + CompletableFuture newRequest = new CompletableFuture<>(); + try { + overrideTableView(channel, serviceUnit, new ServiceUnitStateData(Owned, brokerId1, 1)); + getOwnerRequests.put(serviceUnit, oldRequest); + oldRequest.whenComplete((__, ___) -> getOwnerRequests.put(serviceUnit, newRequest)); + + channel.handleSkippedEvent(serviceUnit); + + assertEquals(oldRequest.getNow(null), brokerId1); + assertTrue(getOwnerRequests.get(serviceUnit) == newRequest, + "A stale skipped-event cleanup must not remove a newer request future"); + } finally { + getOwnerRequests.remove(serviceUnit); + oldRequest.cancel(false); + overrideTableView(channel, serviceUnit, null); + } + } + + @Test(priority = 1) + public void testCompletedCleanupJobDoesNotRemoveNewCleanupJob() { + ServiceUnitStateChannelImpl channel = (ServiceUnitStateChannelImpl) channel1; + String broker = brokerId3; + var cleanupJobs = channel.getCleanupJobs(); + cleanupJobs.remove(broker); + channel.scheduleCleanup(broker, 60L); + CompletableFuture oldJob = cleanupJobs.get(broker); + assertNotNull(oldJob); + + CompletableFuture newJob = null; + try { + // Broker-creation handling removes a cleanup job before cancelling it. A later broker-deletion + // event can therefore schedule a new job before the old job's completion cleanup runs. + assertTrue(cleanupJobs.remove(broker, oldJob)); + channel.scheduleCleanup(broker, 60L); + newJob = cleanupJobs.get(broker); + assertNotNull(newJob); + assertTrue(newJob != oldJob); + + // The previous unconditional removal would remove newJob here. + assertTrue(oldJob.cancel(false)); + + assertTrue(cleanupJobs.get(broker) == newJob, + "A stale cleanup job completion must not remove a newer cleanup job future"); + + assertTrue(newJob.cancel(false)); + assertFalse(cleanupJobs.containsKey(broker), + "The newer cleanup job must remove itself after completion"); + } finally { + cleanupJobs.remove(broker); + oldJob.cancel(false); + if (newJob != null) { + newJob.cancel(false); + } + } + } + @Test(priority = 100) public void channelValidationTest() throws ExecutionException, InterruptedException, IllegalAccessException, PulsarServerException, @@ -822,6 +921,59 @@ public void handleBrokerCreationEventTest() throws IllegalAccessException { } + @Test(priority = 8) + public void handleBrokerCreationEventDoesNotCancelNewCleanupJobTest() { + ServiceUnitStateChannelImpl channel = (ServiceUnitStateChannelImpl) channel1; + var cleanupJobs = channel.getCleanupJobs(); + String broker = brokerId2; + CompletableFuture healthCheck = new CompletableFuture<>(); + cleanupJobs.remove(broker); + channel.scheduleCleanup(broker, 60L); + CompletableFuture oldJob = cleanupJobs.get(broker); + assertNotNull(oldJob); + + reset(brokers); + doReturn(healthCheck).when(brokers).healthcheckAsync(eq(TopicVersion.V2), any()); + doReturn(brokers).when(pulsarAdmin).brokers(); + CompletableFuture newJob = null; + try { + channel.handleBrokerRegistrationEvent(broker, NotificationType.Created); + verify(brokers, times(1)).healthcheckAsync(eq(TopicVersion.V2), any()); + + // The old cleanup can finish while the asynchronous health check is still pending. A later + // broker-deletion event can then schedule a new cleanup job for the same broker. + assertTrue(oldJob.complete(null)); + assertFalse(cleanupJobs.containsKey(broker)); + channel.scheduleCleanup(broker, 60L); + newJob = cleanupJobs.get(broker); + assertNotNull(newJob); + assertTrue(newJob != oldJob); + + healthCheck.complete(null); + + CompletableFuture expectedNewJob = newJob; + Awaitility.await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> { + assertTrue(cleanupJobs.get(broker) == expectedNewJob, + "A stale broker-creation callback must not remove a newer cleanup job"); + assertFalse(expectedNewJob.isCancelled()); + }); + + assertTrue(newJob.cancel(false)); + assertFalse(cleanupJobs.containsKey(broker), + "The newer cleanup job must remove itself after cancellation"); + } finally { + cleanupJobs.remove(broker); + oldJob.cancel(false); + if (newJob != null) { + newJob.cancel(false); + } + reset(brokers); + doReturn(CompletableFuture.failedFuture(new RuntimeException("failed"))).when(brokers) + .healthcheckAsync(eq(TopicVersion.V2), any()); + reset(pulsarAdmin); + } + } + @Test(priority = 9) public void handleBrokerDeletionEventTest() throws Exception { @@ -2075,9 +2227,9 @@ public void testHandleExistingResolvesAssigningStateOnChannelRestart() } } - private static ConcurrentHashMap>> getOwnerRequests( + private static ConcurrentHashMap> getOwnerRequests( ServiceUnitStateChannel channel) throws IllegalAccessException { - return (ConcurrentHashMap>>) + return (ConcurrentHashMap>) FieldUtils.readDeclaredField(channel, "getOwnerRequests", true); } From 25b604a45eae552179e9caee0ec1b5dd5fb2900a Mon Sep 17 00:00:00 2001 From: SongOf <46475785+SongOf@users.noreply.github.com> Date: Sun, 26 Jul 2026 18:41:34 +0800 Subject: [PATCH 150/213] [fix][broker] Release entry on GetLastMessageId when parseMessageMetadata throws (#26089) Co-authored-by: maxlisongsong (cherry picked from commit 4790c386c6d57f2016765f459cad1d77e4f17796) --- .../pulsar/broker/service/ServerCnx.java | 11 +- .../GetLastMessageIdEntryLeakTest.java | 109 ++++++++++++++++++ 2 files changed, 116 insertions(+), 4 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/GetLastMessageIdEntryLeakTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 1a169d7561bad..b23da296586e2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -2446,10 +2446,13 @@ public String toString() { }, null); CompletableFuture batchSizeFuture = entryFuture.thenApply(entry -> { - MessageMetadata metadata = Commands.parseMessageMetadata(entry.getDataBuffer()); - int batchSize = metadata.getNumMessagesInBatch(); - entry.release(); - return metadata.hasNumMessagesInBatch() ? batchSize : -1; + try { + MessageMetadata metadata = Commands.parseMessageMetadata(entry.getDataBuffer()); + int batchSize = metadata.getNumMessagesInBatch(); + return metadata.hasNumMessagesInBatch() ? batchSize : -1; + } finally { + entry.release(); + } }); batchSizeFuture.whenComplete((batchSize, e) -> { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/GetLastMessageIdEntryLeakTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/GetLastMessageIdEntryLeakTest.java new file mode 100644 index 0000000000000..ce612bbbc3355 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/GetLastMessageIdEntryLeakTest.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.spy; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertThrows; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import lombok.Cleanup; +import org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntryCallback; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.EntryImpl; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.Schema; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class GetLastMessageIdEntryLeakTest extends ProducerConsumerBase { + @BeforeMethod + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @AfterMethod(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + /** + * Reproduces the ByteBuf leak in ServerCnx#getLargestBatchIndexWhenPossible: when + * Commands.parseMessageMetadata throws on the entry read for the last position, the entry (and + * its backing ByteBuf) must still be released. + */ + @Test + public void testEntryReleasedWhenParseMetadataThrows() throws Exception { + final String topic = newTopicName(); + + @Cleanup + Producer producer = pulsarClient.newProducer().topic(topic).create(); + producer.send("payload".getBytes()); + + @Cleanup + Consumer consumer = pulsarClient.newConsumer(Schema.BYTES) + .topic(topic).subscriptionName("sub").subscribe(); + + PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic).get(); + + // A 2-byte, non-magic buffer: parseMessageMetadata reads it but fails at readUnsignedInt + // (needs 4 bytes), which is exactly the corrupt-entry case that triggers the leak. + ByteBuf corruptBuf = Unpooled.buffer(2); + corruptBuf.writeShort(0x0000); + + // Build the entry eagerly so its retain happens now; then drop our own ref so the entry + // "owns" the only ref and we can observe whether the broker releases it. + Position lastPosition = ((ManagedLedgerImpl) persistentTopic.getManagedLedger()).getLastConfirmedEntry(); + EntryImpl corruptEntry = + EntryImpl.create(lastPosition.getLedgerId(), lastPosition.getEntryId(), corruptBuf); + corruptBuf.release(); + assertEquals(corruptBuf.refCnt(), 1); + + // Spy the real ManagedLedgerImpl so asyncReadEntry hands back our corrupt entry, while + // getLastPosition() etc. still delegate to the real ledger. + ManagedLedgerImpl spyLedger = spy((ManagedLedgerImpl) persistentTopic.getManagedLedger()); + doAnswer(inv -> { + ReadEntryCallback callback = inv.getArgument(1); + callback.readEntryComplete(corruptEntry, inv.getArgument(2)); + return null; + }).when(spyLedger).asyncReadEntry(any(Position.class), any(ReadEntryCallback.class), any()); + + FieldUtils.writeField(persistentTopic, "ledger", spyLedger, true); + + // The broker fails the request (MetadataError) - that's expected; the point is the buffer. + assertThrows(Exception.class, consumer::getLastMessageId); + + // Before the fix: parseMessageMetadata throws, entry.release() is skipped -> refCnt stays 1. + // After the fix: release runs in finally -> refCnt drops to 0. + assertEquals(corruptBuf.refCnt(), 0, "entry's ByteBuf leaked when parseMessageMetadata threw"); + } + +} From c65b789d7e046d5b32c002b56c1763c7c4155684 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 27 Jul 2026 06:38:21 +0300 Subject: [PATCH 151/213] [fix][meta][branch-4.0] Tolerate concurrent creation of the underreplication LAYOUT node (#26248) --- .../bookkeeper/PulsarLedgerUnderreplicationManager.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java index 1ed465c5c7da0..920b022905ea2 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java @@ -186,7 +186,13 @@ private void checkLayout() throws ReplicationException.CompatibilityException { if (!store.exists(layoutPath).join()) { LedgerRereplicationLayoutFormat.Builder builder = LedgerRereplicationLayoutFormat.newBuilder(); builder.setType(LAYOUT).setVersion(LAYOUT_VERSION); - store.put(layoutPath, builder.build().toString().getBytes(UTF_8), Optional.of(-1L)).join(); + try { + store.put(layoutPath, builder.build().toString().getBytes(UTF_8), Optional.of(-1L)).get(); + } catch (ExecutionException | InterruptedException e) { + if (!(e.getCause() instanceof MetadataStoreException.BadVersionException)) { + throw new RuntimeException(e); + } + } } else { byte[] layoutData = store.get(layoutPath).join().get().getValue(); From e4e2af2b1f3330650ce3737bc04593bfbc164bda Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 4 Aug 2026 18:01:34 +0300 Subject: [PATCH 152/213] [fix][sec][branch-4.2] Upgrade Spring to 7.0.8 (#26270) (cherry picked from commit d8df12dd94ac5da780949b1d0f8c0d1da1607b12) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a2cb40d28d0e8..b94b59ee54898 100644 --- a/pom.xml +++ b/pom.xml @@ -293,7 +293,7 @@ flexible messaging model and an intuitive client API. 3.17.0 1.0 9.1.6 - 6.2.12 + 7.0.8 4.5.14 4.4.16 0.7.7 From 4fc6fee299c22b62eb095f41739cc7197c0d38cf Mon Sep 17 00:00:00 2001 From: Malla Sandeep Date: Wed, 1 Jul 2026 06:22:15 +0530 Subject: [PATCH 153/213] [improve][fn] Standardize log4j2 Root logger configuration to use system property (#26121) (cherry picked from commit 373ab2b32116229314942fafbde62e4004aff52b) --- .../runtime-all/src/main/resources/java_instance_log4j2.xml | 3 +-- .../src/main/resources/kubernetes_instance_log4j2.xml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml b/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml index c6e4b8eeb86eb..271669bb1638b 100644 --- a/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml +++ b/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml @@ -120,10 +120,9 @@ - info + ${sys:pulsar.log.level:-info} ${sys:pulsar.log.appender} - ${sys:pulsar.log.level} diff --git a/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml b/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml index d245fe7714fa1..582905605c956 100644 --- a/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml +++ b/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml @@ -50,10 +50,9 @@ - info + ${sys:pulsar.log.level:-info} Console - ${sys:pulsar.log.level} From 7aa567d93edfd943b89f9871e6edecc2e0573be5 Mon Sep 17 00:00:00 2001 From: SongOf <46475785+SongOf@users.noreply.github.com> Date: Mon, 20 Jul 2026 18:08:34 +0800 Subject: [PATCH 154/213] [fix][broker] Log exception in PulsarMetadataEventSynchronizer failure path (#26203) Co-authored-by: maxlisongsong (cherry picked from commit f1650c5ab57bb9742717852e2496ffcea0f6f7b9) --- .../pulsar/broker/service/PulsarMetadataEventSynchronizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarMetadataEventSynchronizer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarMetadataEventSynchronizer.java index 25af3597c2006..6597dd337ab3d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarMetadataEventSynchronizer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarMetadataEventSynchronizer.java @@ -210,7 +210,7 @@ private void startConsumer() { .collect(Collectors.toList())), c, msg); } } catch (Exception e) { - log.warn("Failed to synchronize {} for {}", msg.getMessageId(), topicName); + log.warn("Failed to synchronize {} for {}", msg.getMessageId(), topicName, e); } }); consumerBuilder.subscribeAsync().thenAccept(consumer -> { From 1be5f2fe1a1be70b07693388f8585da6c5797189 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Jul 2026 22:15:55 +0300 Subject: [PATCH 155/213] [fix][sec] Bump google.golang.org/grpc from 1.79.3 to 1.82.1 in /pulsar-function-go/examples (#26231) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> (cherry picked from commit 99f5848926813fad7d4dde1546b2cfc702394a3d) --- pulsar-function-go/examples/go.mod | 8 +++--- pulsar-function-go/examples/go.sum | 40 +++++++++++++++--------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/pulsar-function-go/examples/go.mod b/pulsar-function-go/examples/go.mod index 3040d08d08187..6e80ff92e5d64 100644 --- a/pulsar-function-go/examples/go.mod +++ b/pulsar-function-go/examples/go.mod @@ -44,12 +44,12 @@ require ( go.uber.org/atomic v1.11.0 // indirect golang.org/x/mod v0.35.0 // indirect golang.org/x/net v0.55.0 // indirect - golang.org/x/oauth2 v0.34.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sys v0.45.0 // indirect golang.org/x/text v0.37.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect - google.golang.org/grpc v1.79.3 // indirect - google.golang.org/protobuf v1.36.10 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect + google.golang.org/grpc v1.82.1 // indirect + google.golang.org/protobuf v1.36.11 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/pulsar-function-go/examples/go.sum b/pulsar-function-go/examples/go.sum index aebbffd2a138e..bbecdf3040d27 100644 --- a/pulsar-function-go/examples/go.sum +++ b/pulsar-function-go/examples/go.sum @@ -180,16 +180,16 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 h1:CV7UdSGJt/Ao6Gp4CXckLxVRRsRgDHoI8XjbL3PDl8s= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0/go.mod h1:FRmFuRJfag1IZ2dPkHnEoSFVgTVPUd2qf5Vi69hLb8I= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= -go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= -go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= -go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= -go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -207,8 +207,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= -golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= -golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -230,14 +230,14 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= -google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= -google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= -google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= -google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From 6a9364759433bf330575b64d070f456bd5d74a37 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 24 Jul 2026 07:15:21 +0300 Subject: [PATCH 156/213] [fix][sec] Upgrade grpc in pulsar-function-go to 1.82.1 to fix GHSA-hrxh-6v49-42gf (#26235) (cherry picked from commit 8aab815314ddcfcef2780f295e0675b42c59fa89) --- pulsar-function-go/go.mod | 8 ++++---- pulsar-function-go/go.sum | 40 +++++++++++++++++++-------------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/pulsar-function-go/go.mod b/pulsar-function-go/go.mod index 54a618c7e32aa..53365d37757e9 100644 --- a/pulsar-function-go/go.mod +++ b/pulsar-function-go/go.mod @@ -9,8 +9,8 @@ require ( github.com/prometheus/client_model v0.6.1 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.10.0 - google.golang.org/grpc v1.79.3 - google.golang.org/protobuf v1.36.10 + google.golang.org/grpc v1.82.1 + google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v2 v2.4.0 ) @@ -50,10 +50,10 @@ require ( go.uber.org/atomic v1.11.0 // indirect golang.org/x/mod v0.35.0 // indirect golang.org/x/net v0.55.0 // indirect - golang.org/x/oauth2 v0.34.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sys v0.45.0 // indirect golang.org/x/text v0.37.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apimachinery v0.32.3 // indirect diff --git a/pulsar-function-go/go.sum b/pulsar-function-go/go.sum index 619d0258f8813..852a5cdd4becb 100644 --- a/pulsar-function-go/go.sum +++ b/pulsar-function-go/go.sum @@ -181,16 +181,16 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 h1:CV7UdSGJt/Ao6Gp4CXckLxVRRsRgDHoI8XjbL3PDl8s= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0/go.mod h1:FRmFuRJfag1IZ2dPkHnEoSFVgTVPUd2qf5Vi69hLb8I= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= -go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= -go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= -go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= -go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -208,8 +208,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= -golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= -golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -231,14 +231,14 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= -google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= -google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= -google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= -google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From c0c85c46da3279ebbcd1d94608a4ff3b0703263f Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 27 Jul 2026 17:13:57 +0800 Subject: [PATCH 157/213] [fix][sec] Upgrade lz4-java to 1.11.1 to address CVE-2026-59949 (#26250) (cherry picked from commit bbd1970deebc9d0d4136cb5d18d60d0c26f9b3ba) Signed-off-by: Zixuan Liu --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b94b59ee54898..bb3867123a952 100644 --- a/pom.xml +++ b/pom.xml @@ -377,7 +377,7 @@ flexible messaging model and an intuitive client API. 1.11.0 2.15.1 2.1.10 - 1.10.3 + 1.11.1 From 454fa556fc8dfdfe12c541082374b7d663638d3b Mon Sep 17 00:00:00 2001 From: void-ptr974 Date: Mon, 27 Jul 2026 17:39:15 +0800 Subject: [PATCH 158/213] [fix][client] Avoid exception in ConsumerImpl hasMessageAvailable before first receive (#25857) Co-authored-by: Lari Hotari (cherry picked from commit ac4cda7fc12adbbe7f86b3a5f3c84340f91829ca) --- .../api/SimpleProducerConsumerTest.java | 132 ++++++++++++++++++ .../pulsar/client/impl/ConsumerImpl.java | 72 ++++++---- 2 files changed, 175 insertions(+), 29 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java index c553950014f6e..e09b3b6047f26 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java @@ -21,6 +21,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.pulsar.common.naming.TopicName.PARTITIONED_TOPIC_SUFFIX; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.any; import static org.mockito.Mockito.atLeastOnce; @@ -3540,6 +3541,137 @@ public void testConsumerSubscriptionInitialize() throws Exception { log.info("-- Exiting {} test --", methodName); } + @Test(timeOut = 100000) + public void testConsumerImplHasMessageAvailableDoesNotThrowBeforeReceive() throws Exception { + log.info("-- Starting {} test --", methodName); + String topicName = BrokerTestUtil.newUniqueName("persistent://my-property/my-ns/has-message-available"); + + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(topicName) + .enableBatching(false) + .create(); + producer.send("existing-message".getBytes(UTF_8)); + + // hasMessageAvailable is exposed by ConsumerImpl, not the Consumer public API. + @Cleanup + ConsumerImpl latestConsumer = (ConsumerImpl) pulsarClient.newConsumer() + .topic(topicName) + .subscriptionName("test-has-message-available-latest") + .receiverQueueSize(0) + .subscribe(); + assertThat(latestConsumer.hasMessageAvailableAsync()) + .succeedsWithin(Duration.ofSeconds(10)) + .isEqualTo(false); + assertThatCode(latestConsumer::hasMessageAvailable) + .doesNotThrowAnyException(); + assertFalse(latestConsumer.hasMessageAvailable()); + + @Cleanup + ConsumerImpl earliestConsumer = (ConsumerImpl) pulsarClient.newConsumer() + .topic(topicName) + .subscriptionName("test-has-message-available-earliest") + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .receiverQueueSize(0) + .subscribe(); + assertThat(earliestConsumer.hasMessageAvailableAsync()) + .succeedsWithin(Duration.ofSeconds(10)) + .isEqualTo(true); + assertThatCode(earliestConsumer::hasMessageAvailable) + .doesNotThrowAnyException(); + assertTrue(earliestConsumer.hasMessageAvailable()); + + log.info("-- Exiting {} test --", methodName); + } + + @Test(timeOut = 100000) + public void testReaderHasMessageAvailableDoesNotThrowBeforeRead() throws Exception { + log.info("-- Starting {} test --", methodName); + String topicName = BrokerTestUtil.newUniqueName("persistent://my-property/my-ns/reader-has-message-available"); + + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(topicName) + .enableBatching(false) + .create(); + producer.send("existing-message".getBytes(UTF_8)); + + @Cleanup + Reader latestReader = pulsarClient.newReader() + .topic(topicName) + .startMessageId(MessageId.latest) + .receiverQueueSize(0) + .create(); + assertThat(latestReader.hasMessageAvailableAsync()) + .succeedsWithin(Duration.ofSeconds(10)) + .isEqualTo(false); + assertThatCode(latestReader::hasMessageAvailable) + .doesNotThrowAnyException(); + assertFalse(latestReader.hasMessageAvailable()); + + @Cleanup + Reader earliestReader = pulsarClient.newReader() + .topic(topicName) + .startMessageId(MessageId.earliest) + .receiverQueueSize(0) + .create(); + assertThat(earliestReader.hasMessageAvailableAsync()) + .succeedsWithin(Duration.ofSeconds(10)) + .isEqualTo(true); + assertThatCode(earliestReader::hasMessageAvailable) + .doesNotThrowAnyException(); + assertTrue(earliestReader.hasMessageAvailable()); + + log.info("-- Exiting {} test --", methodName); + } + + @Test(timeOut = 100000) + public void testHasMessageAvailableDoesNotThrowOnEmptyTopic() throws Exception { + log.info("-- Starting {} test --", methodName); + String topicName = BrokerTestUtil.newUniqueName("persistent://my-property/my-ns/empty-has-message-available"); + + @Cleanup + ConsumerImpl consumer = (ConsumerImpl) pulsarClient.newConsumer() + .topic(topicName) + .subscriptionName("test-empty-has-message-available") + .receiverQueueSize(0) + .subscribe(); + assertThat(consumer.hasMessageAvailableAsync()) + .succeedsWithin(Duration.ofSeconds(10)) + .isEqualTo(false); + assertThatCode(consumer::hasMessageAvailable) + .doesNotThrowAnyException(); + assertFalse(consumer.hasMessageAvailable()); + + @Cleanup + Reader latestReader = pulsarClient.newReader() + .topic(topicName) + .startMessageId(MessageId.latest) + .receiverQueueSize(0) + .create(); + assertThat(latestReader.hasMessageAvailableAsync()) + .succeedsWithin(Duration.ofSeconds(10)) + .isEqualTo(false); + assertThatCode(latestReader::hasMessageAvailable) + .doesNotThrowAnyException(); + assertFalse(latestReader.hasMessageAvailable()); + + @Cleanup + Reader earliestReader = pulsarClient.newReader() + .topic(topicName) + .startMessageId(MessageId.earliest) + .receiverQueueSize(0) + .create(); + assertThat(earliestReader.hasMessageAvailableAsync()) + .succeedsWithin(Duration.ofSeconds(10)) + .isEqualTo(false); + assertThatCode(earliestReader::hasMessageAvailable) + .doesNotThrowAnyException(); + assertFalse(earliestReader.hasMessageAvailable()); + + log.info("-- Exiting {} test --", methodName); + } + @Test(timeOut = 100000) public void testMultiTopicsConsumerImplPauseForPartitionNumberChange() throws Exception { log.info("-- Starting {} test --", methodName); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 87bb936d181cd..46897f2950ee4 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -2705,6 +2705,19 @@ public CompletableFuture hasMessageAvailableAsync() { // we haven't read yet. use startMessageId for comparison if (lastDequeuedMessageId == MessageId.earliest) { + if (startMessageId == null) { + internalGetLastMessageIdAsync().thenAccept(response -> { + lastMessageIdInBroker = response.lastMessageId; + completehasMessageAvailableWithValue(booleanFuture, + hasMoreMessagesThanMarkDeletePosition(response, false, false)); + }).exceptionally(e -> { + log.error("[{}][{}] Failed getLastMessageId command", topic, subscription, e); + booleanFuture.completeExceptionally(FutureUtil.unwrapCompletionException(e)); + return null; + }); + return booleanFuture; + } + // If the last seek is called with timestamp, startMessageId cannot represent the position to start, so we // have to get the mark-delete position from the GetLastMessageId response to compare as well. // if we are starting from latest, we should seek to the actual last message first. @@ -2720,33 +2733,12 @@ public CompletableFuture hasMessageAvailableAsync() { } future.thenAccept(response -> { - MessageIdAdv lastMessageId = (MessageIdAdv) response.lastMessageId; - MessageIdAdv markDeletePosition = (MessageIdAdv) response.markDeletePosition; - - if (markDeletePosition != null && !(markDeletePosition.getEntryId() < 0 - && markDeletePosition.getLedgerId() > lastMessageId.getLedgerId())) { - // we only care about comparing ledger ids and entry ids as mark delete position doesn't have - // other ids such as batch index - int result = ComparisonChain.start() - .compare(markDeletePosition.getLedgerId(), lastMessageId.getLedgerId()) - .compare(markDeletePosition.getEntryId(), lastMessageId.getEntryId()) - .result(); - if (lastMessageId.getEntryId() < 0) { - completehasMessageAvailableWithValue(booleanFuture, false); - } else if (hasSoughtByTimestamp) { - completehasMessageAvailableWithValue(booleanFuture, result < 0); - } else { - completehasMessageAvailableWithValue(booleanFuture, - resetIncludeHead ? result <= 0 : result < 0); - } - } else if (lastMessageId == null || lastMessageId.getEntryId() < 0) { - completehasMessageAvailableWithValue(booleanFuture, false); - } else { - completehasMessageAvailableWithValue(booleanFuture, resetIncludeHead); - } + completehasMessageAvailableWithValue(booleanFuture, + hasMoreMessagesThanMarkDeletePosition(response, + !hasSoughtByTimestamp && resetIncludeHead, resetIncludeHead)); }).exceptionally(ex -> { log.error("[{}][{}] Failed getLastMessageId command", topic, subscription, ex); - booleanFuture.completeExceptionally(ex.getCause()); + booleanFuture.completeExceptionally(FutureUtil.unwrapCompletionException(ex)); return null; }); @@ -2763,8 +2755,8 @@ public CompletableFuture hasMessageAvailableAsync() { completehasMessageAvailableWithValue(booleanFuture, hasMoreMessages(lastMessageIdInBroker, startMessageId, resetIncludeHead)); }).exceptionally(e -> { - log.error("[{}][{}] Failed getLastMessageId command", topic, subscription); - booleanFuture.completeExceptionally(e.getCause()); + log.error("[{}][{}] Failed getLastMessageId command", topic, subscription, e); + booleanFuture.completeExceptionally(FutureUtil.unwrapCompletionException(e)); return null; }); @@ -2780,8 +2772,8 @@ public CompletableFuture hasMessageAvailableAsync() { completehasMessageAvailableWithValue(booleanFuture, hasMoreMessages(lastMessageIdInBroker, lastDequeuedMessageId, false)); }).exceptionally(e -> { - log.error("[{}][{}] Failed getLastMessageId command", topic, subscription); - booleanFuture.completeExceptionally(e.getCause()); + log.error("[{}][{}] Failed getLastMessageId command", topic, subscription, e); + booleanFuture.completeExceptionally(FutureUtil.unwrapCompletionException(e)); return null; }); } @@ -2805,6 +2797,28 @@ private boolean hasMoreMessages(MessageId lastMessageIdInBroker, MessageId messa && ((MessageIdImpl) lastMessageIdInBroker).getEntryId() != -1; } + private boolean hasMoreMessagesThanMarkDeletePosition(GetLastMessageIdResponse response, boolean inclusive, + boolean includeHeadWhenMarkDeleteIsAfterLastMessage) { + MessageIdAdv lastMessageId = (MessageIdAdv) response.lastMessageId; + if (lastMessageId == null || lastMessageId.getEntryId() < 0) { + return false; + } + + MessageIdAdv markDeletePosition = (MessageIdAdv) response.markDeletePosition; + if (markDeletePosition == null || (markDeletePosition.getEntryId() < 0 + && markDeletePosition.getLedgerId() > lastMessageId.getLedgerId())) { + return includeHeadWhenMarkDeleteIsAfterLastMessage; + } + + // We only care about comparing ledger ids and entry ids as mark delete position doesn't have + // other ids such as batch index. + int result = ComparisonChain.start() + .compare(markDeletePosition.getLedgerId(), lastMessageId.getLedgerId()) + .compare(markDeletePosition.getEntryId(), lastMessageId.getEntryId()) + .result(); + return inclusive ? result <= 0 : result < 0; + } + private static final class GetLastMessageIdResponse { final MessageId lastMessageId; final MessageId markDeletePosition; From e122ab0a9518606315005bc8ebd76a4a01632e2d Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 27 Jul 2026 10:17:48 +0300 Subject: [PATCH 159/213] [fix][ml] Tolerate concurrent creation of the managed ledger z-node (#26247) (cherry picked from commit d54d83df25ca3ebdb7574a80f31e5252d5d91995) (cherry picked from commit cd889206c1be9e50c2238c699fbdce529014251c) --- .../mledger/impl/MetaStoreImpl.java | 11 +++++ .../mledger/impl/MetaStoreImplTest.java | 48 +++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java index 611d9d60202cd..7b122691fe46f 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java @@ -126,6 +126,17 @@ public void getManagedLedgerInfo(String ledgerName, boolean createIfMissing, Map } callback.operationComplete(ledgerBuilder.build(), stat); }).exceptionally(ex -> { + if (FutureUtil.unwrapCompletionException(ex) + instanceof MetadataStoreException.BadVersionException) { + // The z-node was created concurrently after the read above returned + // "not found". This happens for example when the broker creates the + // partitions of a topic while the same topic is being loaded. Read the + // z-node back instead of failing the managed ledger initialization. + log.info("Managed ledger path '{}' was concurrently created, reading it " + + "back", path); + getManagedLedgerInfo(ledgerName, false, null, callback); + return null; + } callback.operationFailed(getException(ex)); return null; }); diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/MetaStoreImplTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/MetaStoreImplTest.java index 00ad494a5ca59..3f0ac3aab2cd1 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/MetaStoreImplTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/MetaStoreImplTest.java @@ -18,6 +18,10 @@ */ package org.apache.bookkeeper.mledger.impl; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.spy; +import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; @@ -25,6 +29,7 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import lombok.AllArgsConstructor; import lombok.Data; @@ -38,6 +43,7 @@ import org.apache.pulsar.metadata.api.MetadataCache; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.Stat; +import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.apache.pulsar.metadata.impl.FaultInjectionMetadataStore; import org.testng.annotations.Test; @@ -159,6 +165,48 @@ public void operationComplete(ManagedLedgerInfo result, Stat version) { promise.get(); } + /** + * The z-node backing a managed ledger can be created by another party (for example + * {@code AdminResource#tryCreatePartitionAsync}) in between the read and the write that + * {@code getManagedLedgerInfo(createIfMissing = true)} performs. The metadata store reports that as a + * {@link MetadataStoreException.BadVersionException}, which must not fail the managed ledger initialization. + */ + @Test(timeOut = 20000) + void createMLNodeConcurrently() throws Exception { + String ledgerName = "my_test"; + String path = "/managed-ledgers/" + ledgerName; + + AtomicBoolean raceInjected = new AtomicBoolean(); + MetadataStoreExtended racyStore = spy(metadataStore); + doAnswer(invocation -> { + if (path.equals(invocation.getArgument(0)) && raceInjected.compareAndSet(false, true)) { + // Create the z-node behind the caller's back and report it as still missing, so that the + // subsequent create hits an already existing z-node. + metadataStore.put(path, new byte[0], Optional.of(-1L)).get(); + return CompletableFuture.completedFuture(Optional.empty()); + } + return invocation.callRealMethod(); + }).when(racyStore).get(anyString()); + + MetaStore store = new MetaStoreImpl(racyStore, executor); + + final CompletableFuture promise = new CompletableFuture<>(); + store.getManagedLedgerInfo(ledgerName, true, new MetaStoreCallback() { + public void operationFailed(MetaStoreException e) { + promise.completeExceptionally(e); + } + + public void operationComplete(ManagedLedgerInfo result, Stat version) { + promise.complete(result); + } + }); + + ManagedLedgerInfo info = promise.get(); + assertNotNull(info); + assertEquals(info.getLedgerInfoCount(), 0); + assertTrue(raceInjected.get()); + } + @Test(timeOut = 20000) void updatingCursorNode() throws Exception { MetaStore store = new MetaStoreImpl(metadataStore, executor); From 44bbc55334ba4a42876290d43fae65c41c564eef Mon Sep 17 00:00:00 2001 From: void-ptr974 Date: Wed, 29 Jul 2026 15:19:34 +0800 Subject: [PATCH 160/213] [fix][broker] Fix delayed-delivery bucket merge failures when delayedDeliveryMaxNumBuckets is 1-3 (#26242) (cherry picked from commit a107d5146222913236dba991c8b65e9a1ed444d0) --- .../bucket/BucketDelayedDeliveryTracker.java | 23 +++-- .../BucketDelayedDeliveryTrackerTest.java | 99 +++++++++++++++++++ 2 files changed, 114 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index 68ce805c0add2..0091f6a0f0266 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -424,13 +424,17 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver return true; } - private synchronized List selectMergedBuckets(final List values, int mergeNum) { - checkArgument(mergeNum < values.size()); + @VisibleForTesting + synchronized List selectMergedBuckets(final List values, int mergeNum) { + if (values.size() < 2 || mergeNum < 2) { + return Collections.emptyList(); + } + int actualMergeNum = Math.min(mergeNum, values.size()); long minNumberMessages = Long.MAX_VALUE; long minScheduleTimestamp = Long.MAX_VALUE; int minIndex = -1; - for (int i = 0; i + (mergeNum - 1) < values.size(); i++) { - List immutableBuckets = values.subList(i, i + mergeNum); + for (int i = 0; i + (actualMergeNum - 1) < values.size(); i++) { + List immutableBuckets = values.subList(i, i + actualMergeNum); if (immutableBuckets.stream().allMatch(bucket -> { // We should skip the bucket which last segment already been load to memory, // avoid record replicated index. @@ -441,8 +445,11 @@ private synchronized List selectMergedBuckets(final List bucket.firstScheduleTimestamps.get(bucket.currentSegmentEntryId + 1)) + .mapToLong(bucket -> bucket.firstScheduleTimestamps.get(bucket.currentSegmentEntryId)) .min().getAsLong(); if (scheduleTimestamp < minScheduleTimestamp) { minScheduleTimestamp = scheduleTimestamp; @@ -453,9 +460,9 @@ private synchronized List selectMergedBuckets(final List= 0) { - return values.subList(minIndex, minIndex + mergeNum); - } else if (mergeNum > 2){ - return selectMergedBuckets(values, mergeNum - 1); + return values.subList(minIndex, minIndex + actualMergeNum); + } else if (actualMergeNum > 2) { + return selectMergedBuckets(values, actualMergeNum - 1); } else { return Collections.emptyList(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java index 14a257f303d0d..231e3690a6038 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java @@ -33,6 +33,7 @@ import java.lang.reflect.Method; import java.nio.ByteBuffer; import java.time.Clock; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.NavigableMap; @@ -556,6 +557,18 @@ public CompletableFuture deleteBucketSnapshot(long bucketId) { } } + private ImmutableBucket createMergeableBucket(TrackerWithStorage trackerWithStorage, long startLedgerId, + long endLedgerId, List firstScheduleTimestamps) { + MutableBucket mutableBucket = trackerWithStorage.tracker.getLastMutableBucket(); + ImmutableBucket bucket = new ImmutableBucket(mutableBucket.dispatcherName, mutableBucket.cursor, + mutableBucket.sequencer, mutableBucket.bucketSnapshotStorage, startLedgerId, endLedgerId); + bucket.setCurrentSegmentEntryId(1); + bucket.setLastSegmentEntryId(firstScheduleTimestamps.size()); + bucket.setFirstScheduleTimestamps(firstScheduleTimestamps); + bucket.setNumberBucketDelayedMessages(1); + return bucket; + } + private TrackerWithStorage createTrackerWithMockLedger(long firstLedgerId, int maxNumBuckets) throws Exception { return createTrackerWithMockLedger(firstLedgerId, maxNumBuckets, new MockBucketSnapshotStorage()); @@ -597,6 +610,48 @@ public Position getMarkDeletedPosition() { return new TrackerWithStorage(tracker, storage, mockClockTime); } + @DataProvider(name = "smallMaxNumBuckets") + private Object[][] smallMaxNumBuckets() { + return new Object[][]{{1}, {2}, {3}}; + } + + @Test(dataProvider = "smallMaxNumBuckets") + public void testMergeSupportsSmallMaxNumBuckets(int maxNumBuckets) throws Exception { + TrackerWithStorage ts = createTrackerWithMockLedger(0L, maxNumBuckets); + int messageCount = (maxNumBuckets + 1) * 5 + 1; + NavigableSet expectedMessages = new TreeSet<>(); + try { + for (int i = 1; i <= messageCount; i++) { + assertTrue(ts.tracker.addMessage(i, i, i % 5 == 0 ? 20L : 10L)); + expectedMessages.add(PositionFactory.create(i, i)); + } + + Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> { + synchronized (ts.tracker) { + List buckets = List.copyOf( + ts.tracker.getImmutableBuckets().asMapOfRanges().values()); + assertTrue(!buckets.isEmpty()); + assertTrue(buckets.size() <= maxNumBuckets); + assertTrue(buckets.stream().noneMatch(bucket -> bucket.merging + || bucket.getSnapshotCreateFuture() + .map(future -> !future.isDone() || future.isCompletedExceptionally()) + .orElse(true))); + } + }); + + ts.clockTime.set(20L); + List scheduledMessages = new ArrayList<>(); + Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> { + scheduledMessages.addAll(ts.tracker.getScheduledMessages(expectedMessages.size())); + assertEquals(scheduledMessages.size(), expectedMessages.size()); + }); + assertEquals(new TreeSet<>(scheduledMessages), expectedMessages); + assertEquals(ts.tracker.getNumberOfDelayedMessages(), 0L); + } finally { + ts.close(); + } + } + @Test public void testTrimRemovesOrphanedBuckets() throws Exception { long firstLedgerId = 31L; @@ -628,6 +683,50 @@ public void testTrimRemovesOrphanedBuckets() throws Exception { ts.close(); } + @Test + public void testSelectMergedBucketsSupportsTwoBuckets() throws Exception { + TrackerWithStorage ts = createTrackerWithMockLedger(0L, 1); + try { + ImmutableBucket firstBucket = createMergeableBucket(ts, 1L, 1L, List.of(10L, 20L)); + ImmutableBucket secondBucket = createMergeableBucket(ts, 2L, 2L, List.of(10L, 20L)); + + assertEquals(ts.tracker.selectMergedBuckets(List.of(firstBucket, secondBucket), 4), + List.of(firstBucket, secondBucket)); + } finally { + ts.close(); + } + } + + @Test + public void testSelectMergedBucketsHandlesOneUnloadedSegment() throws Exception { + TrackerWithStorage ts = createTrackerWithMockLedger(0L, 1); + try { + ImmutableBucket firstBucket = createMergeableBucket(ts, 1L, 1L, List.of(10L, 20L)); + ImmutableBucket secondBucket = createMergeableBucket(ts, 2L, 2L, List.of(10L, 20L)); + ImmutableBucket thirdBucket = createMergeableBucket(ts, 3L, 3L, List.of(10L, 20L)); + + assertEquals(ts.tracker.selectMergedBuckets(List.of(firstBucket, secondBucket, thirdBucket), 2), + List.of(firstBucket, secondBucket)); + } finally { + ts.close(); + } + } + + @Test + public void testSelectMergedBucketsUsesNextUnloadedSegmentTimestamp() throws Exception { + TrackerWithStorage ts = createTrackerWithMockLedger(0L, 1); + try { + ImmutableBucket firstBucket = createMergeableBucket(ts, 1L, 1L, List.of(10L, 100L, 10L)); + ImmutableBucket secondBucket = createMergeableBucket(ts, 2L, 2L, List.of(10L, 100L, 10L)); + ImmutableBucket thirdBucket = createMergeableBucket(ts, 3L, 3L, List.of(10L, 50L, 1000L)); + + assertEquals(ts.tracker.selectMergedBuckets(List.of(firstBucket, secondBucket, thirdBucket), 2), + List.of(secondBucket, thirdBucket)); + } finally { + ts.close(); + } + } + @Test public void testTrimDoesNotDeleteBucketOverlappingFirstActiveLedger() throws Exception { RecordingDeleteStorage storage = new RecordingDeleteStorage(); From 75ae9b34ca8c7d829fcd0e97f5ed324e9a4420cc Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 7 Aug 2026 16:44:19 +0300 Subject: [PATCH 161/213] [fix][misc] Remove unnecessary " {}" suffix from log lines when there are no context attributes (#26285) (cherry picked from commit af54f64e1880e32d859e6f5f726503f7228f9804) --- buildtools/src/main/resources/log4j2.xml | 2 +- conf/functions_log4j2.xml | 6 +++--- conf/log4j2.yaml | 6 +++--- microbench/src/main/resources/log4j2.xml | 2 +- pulsar-broker/src/test/resources/log4j2.xml | 2 +- pulsar-client-admin/src/test/resources/log4j2.xml | 2 +- pulsar-functions/localrun/src/main/resources/log4j2.xml | 2 +- .../runtime-all/src/main/resources/java_instance_log4j2.xml | 6 +++--- .../src/main/resources/kubernetes_instance_log4j2.xml | 2 +- pulsar-proxy/src/test/resources/log4j2.xml | 2 +- tiered-storage/jcloud/src/test/resources/log4j2-test.yml | 4 ++-- 11 files changed, 18 insertions(+), 18 deletions(-) diff --git a/buildtools/src/main/resources/log4j2.xml b/buildtools/src/main/resources/log4j2.xml index b0d01a734c518..62d98f208ebaf 100644 --- a/buildtools/src/main/resources/log4j2.xml +++ b/buildtools/src/main/resources/log4j2.xml @@ -22,7 +22,7 @@ - + diff --git a/conf/functions_log4j2.xml b/conf/functions_log4j2.xml index b4c36867e2d42..364fa0c21cfb9 100644 --- a/conf/functions_log4j2.xml +++ b/conf/functions_log4j2.xml @@ -40,7 +40,7 @@ Console SYSTEM_OUT - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg %X%n + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%equals{ %X}{ {}}{}%n @@ -49,7 +49,7 @@ ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz true - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg %X%n + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%equals{ %X}{ {}}{}%n @@ -82,7 +82,7 @@ ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz true - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg %X%n + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%equals{ %X}{ {}}{}%n diff --git a/conf/log4j2.yaml b/conf/log4j2.yaml index 52b5e92830851..6d28eaf5fdcbf 100644 --- a/conf/log4j2.yaml +++ b/conf/log4j2.yaml @@ -58,7 +58,7 @@ Configuration: - name: Console target: SYSTEM_OUT PatternLayout: - Pattern: "%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg %X%n" + Pattern: "%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%equals{ %X}{ {}}{}%n" - name: ConsoleJson target: SYSTEM_OUT JsonTemplateLayout: @@ -71,7 +71,7 @@ Configuration: filePattern: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}-%d{MM-dd-yyyy}-%i.log.gz" immediateFlush: ${sys:pulsar.log.immediateFlush} PatternLayout: - Pattern: "%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg %X%n" + Pattern: "%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%equals{ %X}{ {}}{}%n" Policies: TimeBasedTriggeringPolicy: interval: 1 @@ -109,7 +109,7 @@ Configuration: fileName : "${sys:pulsar.log.dir}/functions/${ctx:function}/${ctx:functionname}-${ctx:instance}.log" filePattern : "${sys:pulsar.log.dir}/functions/${sys:pulsar.log.file}-${ctx:instance}-%d{MM-dd-yyyy}-%i.log.gz" PatternLayout: - Pattern: "%d{ABSOLUTE} %level{length=5} [%thread] [instance: %X{instance}] %logger{1} - %msg %X%n" + Pattern: "%d{ABSOLUTE} %level{length=5} [%thread] [instance:%X{instance}] %logger{1} - %msg%equals{ %X}{ {}}{}%n" Policies: TimeBasedTriggeringPolicy: interval: 1 diff --git a/microbench/src/main/resources/log4j2.xml b/microbench/src/main/resources/log4j2.xml index 64b03ab62f43c..0898d4d66e489 100644 --- a/microbench/src/main/resources/log4j2.xml +++ b/microbench/src/main/resources/log4j2.xml @@ -22,7 +22,7 @@ - + diff --git a/pulsar-broker/src/test/resources/log4j2.xml b/pulsar-broker/src/test/resources/log4j2.xml index b348a1d04b797..543d35e115319 100644 --- a/pulsar-broker/src/test/resources/log4j2.xml +++ b/pulsar-broker/src/test/resources/log4j2.xml @@ -25,7 +25,7 @@ - + diff --git a/pulsar-client-admin/src/test/resources/log4j2.xml b/pulsar-client-admin/src/test/resources/log4j2.xml index 9b57b450ffa43..bfe1958cd9d12 100644 --- a/pulsar-client-admin/src/test/resources/log4j2.xml +++ b/pulsar-client-admin/src/test/resources/log4j2.xml @@ -25,7 +25,7 @@ - + diff --git a/pulsar-functions/localrun/src/main/resources/log4j2.xml b/pulsar-functions/localrun/src/main/resources/log4j2.xml index d29e798e4797a..57b6e9e7d33c7 100644 --- a/pulsar-functions/localrun/src/main/resources/log4j2.xml +++ b/pulsar-functions/localrun/src/main/resources/log4j2.xml @@ -22,7 +22,7 @@ - + diff --git a/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml b/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml index 271669bb1638b..49336daca9075 100644 --- a/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml +++ b/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml @@ -40,7 +40,7 @@ Console SYSTEM_OUT - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg %X%n + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%equals{ %X}{ {}}{}%n @@ -49,7 +49,7 @@ ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz true - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg %X%n + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%equals{ %X}{ {}}{}%n @@ -82,7 +82,7 @@ ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz true - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg %X%n + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%equals{ %X}{ {}}{}%n diff --git a/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml b/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml index 582905605c956..4b26c51f2024a 100644 --- a/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml +++ b/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml @@ -36,7 +36,7 @@ Console SYSTEM_OUT - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg %X%n + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%equals{ %X}{ {}}{}%n diff --git a/pulsar-proxy/src/test/resources/log4j2.xml b/pulsar-proxy/src/test/resources/log4j2.xml index 261bd2edf6980..de6ad83da0389 100644 --- a/pulsar-proxy/src/test/resources/log4j2.xml +++ b/pulsar-proxy/src/test/resources/log4j2.xml @@ -25,7 +25,7 @@ - + diff --git a/tiered-storage/jcloud/src/test/resources/log4j2-test.yml b/tiered-storage/jcloud/src/test/resources/log4j2-test.yml index 77b3baa97a970..7371e668f9210 100644 --- a/tiered-storage/jcloud/src/test/resources/log4j2-test.yml +++ b/tiered-storage/jcloud/src/test/resources/log4j2-test.yml @@ -33,12 +33,12 @@ Configuration: name: STDOUT target: SYSTEM_OUT PatternLayout: - Pattern: "%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg %X%n" + Pattern: "%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%equals{ %X}{ {}}{}%n" File: name: File fileName: ${filename} PatternLayout: - Pattern: "%d %p %c{1.} [%t] %m%n" + Pattern: "%d %p %c{1.} [%t] %msg%equals{ %X}{ {}}{}%n" Filters: ThresholdFilter: level: error From 8b5ce63cf05ab7a15d0c350d64d62c826d668294 Mon Sep 17 00:00:00 2001 From: void-ptr974 Date: Fri, 7 Aug 2026 12:32:13 +0800 Subject: [PATCH 162/213] [fix][admin] Avoid creating subscriptions when peeking messages if auto-creation is disabled (#26279) (cherry picked from commit 2fbf416e39cb506fbe0b77fc1a92dd5facb80293) --- .../admin/impl/PersistentTopicsBase.java | 44 ++++++++++++------- .../broker/admin/v2/PersistentTopics.java | 4 +- .../broker/admin/PersistentTopicsTest.java | 12 +++++ .../apache/pulsar/client/admin/Topics.java | 4 ++ 4 files changed, 48 insertions(+), 16 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 14e4b03756a95..d56758f75ef2b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -3034,9 +3034,8 @@ protected CompletableFuture internalPeekNthMessageAsync(String subName PersistentReplicator repl = getReplicatorReference(subName, (PersistentTopic) topic); entry = repl.peekNthMessage(messagePosition); } else { - PersistentSubscription sub = - (PersistentSubscription) getSubscriptionReference(subName, (PersistentTopic) topic); - entry = sub.peekNthMessage(messagePosition); + entry = findOrCreateSubscriptionAsync(subName, (PersistentTopic) topic) + .thenCompose(sub -> sub.peekNthMessage(messagePosition)); } } return entry.thenApply(e -> Pair.of(e, (PersistentTopic) topic)); @@ -4603,20 +4602,35 @@ private CompletableFuture topicNotFoundReasonAsync(TopicName topicName) { } /** - * Get the Subscription object reference from the Topic reference. + * Find the subscription or create it when automatic subscription creation is allowed. */ - private Subscription getSubscriptionReference(String subName, PersistentTopic topic) { - try { - Subscription sub = topic.getSubscription(subName); - if (sub == null) { - sub = topic.createSubscription(subName, - InitialPosition.Earliest, false, null).get(); - } - - return checkNotNull(sub); - } catch (Exception e) { - throw new RestException(Status.NOT_FOUND, getSubNotFoundErrorMessage(topicName.toString(), subName)); + private CompletableFuture findOrCreateSubscriptionAsync(String subName, PersistentTopic topic) { + Subscription subscription = topic.getSubscription(subName); + if (subscription != null) { + return CompletableFuture.completedFuture(subscription); } + + return pulsar().getBrokerService().isAllowAutoSubscriptionCreationAsync(topicName) + .thenCompose(isAllowed -> { + Subscription existingSubscription = topic.getSubscription(subName); + if (existingSubscription != null) { + return CompletableFuture.completedFuture(existingSubscription); + } + if (!isAllowed) { + return CompletableFuture.failedFuture(new RestException(Status.PRECONDITION_FAILED, + String.format("Subscription %s does not exist for topic %s and automatic " + + "subscription creation is disabled", + subName, topicName))); + } + return topic.createSubscription(subName, InitialPosition.Earliest, false, null) + .handle((createdSubscription, ex) -> { + if (ex != null || createdSubscription == null) { + throw new RestException(Status.NOT_FOUND, + getSubNotFoundErrorMessage(topicName.toString(), subName)); + } + return createdSubscription; + }); + }); } /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java index 3706194d93199..e06d76918b20c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java @@ -2007,7 +2007,9 @@ public void resetCursorOnPosition( @ApiResponse(code = 404, message = "Namespace or topic, subscription or the message position does not" + " exist"), @ApiResponse(code = 405, message = "Skipping messages on a non-persistent topic is not allowed"), - @ApiResponse(code = 412, message = "Topic name is not valid"), + @ApiResponse(code = 412, + message = "Topic name is not valid, or the subscription does not exist and automatic " + + "subscription creation is disabled"), @ApiResponse(code = 500, message = "Internal server error"), @ApiResponse(code = 503, message = "Failed to validate global cluster configuration")}) public void peekNthMessage( diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index 626805a7032a7..5b465c6846f7e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -100,6 +100,7 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.policies.data.AuthAction; +import org.apache.pulsar.common.policies.data.AutoSubscriptionCreationOverride; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; import org.apache.pulsar.common.policies.data.Policies; @@ -1146,9 +1147,20 @@ public void testPeekWithSubscriptionNameNotExist() throws Exception { producer.send("test" + i); } + admin.namespaces().setAutoSubscriptionCreation("tenant-xyz/ns-abc", + AutoSubscriptionCreationOverride.builder().allowAutoSubscriptionCreation(false).build()); + PulsarAdminException.PreconditionFailedException exception = Assert.expectThrows( + PulsarAdminException.PreconditionFailedException.class, + () -> admin.topics().peekMessages(partitionedTopic, subscriptionName, 3)); + Assert.assertEquals(exception.getStatusCode(), Response.Status.PRECONDITION_FAILED.getStatusCode()); + Assert.assertTrue(admin.topics().getSubscriptions(partitionedTopic).isEmpty()); + + admin.namespaces().setAutoSubscriptionCreation("tenant-xyz/ns-abc", + AutoSubscriptionCreationOverride.builder().allowAutoSubscriptionCreation(true).build()); List> messages = admin.topics().peekMessages(partitionedTopic, subscriptionName, 3); Assert.assertEquals(messages.size(), 3); + Assert.assertEquals(admin.topics().getSubscriptions(partitionedTopic), List.of(subscriptionName)); producer.close(); } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java index a2fcd60deb5a6..8cc577d8c4641 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java @@ -1499,6 +1499,8 @@ PartitionedTopicInternalStats getPartitionedInternalStats(String topic) * Don't have admin permission * @throws NotFoundException * Topic or subscription does not exist + * @throws PreconditionFailedException + * Subscription does not exist and automatic subscription creation is disabled * @throws PulsarAdminException * Unexpected error */ @@ -1532,6 +1534,8 @@ PartitionedTopicInternalStats getPartitionedInternalStats(String topic) * Don't have admin permission * @throws NotFoundException * Topic or subscription does not exist + * @throws PreconditionFailedException + * Subscription does not exist and automatic subscription creation is disabled * @throws PulsarAdminException * Unexpected error */ From fab20c4468feef1a44177714c945392b27afee8c Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Mon, 10 Aug 2026 04:27:50 +0800 Subject: [PATCH 163/213] [fix][txn] Fix X-Pulsar-txn-uncommitted header semantics and add X-Pulsar-txn-consumable for peek messages (#26073) (cherry picked from commit 17de5567020bd4dfa37d082d92d7f8d0aeb9f61b) --- .../admin/impl/PersistentTopicsBase.java | 18 +- .../service/persistent/PersistentTopic.java | 4 + .../transaction/buffer/TransactionBuffer.java | 8 + .../buffer/impl/InMemTransactionBuffer.java | 10 ++ .../buffer/impl/TopicTransactionBuffer.java | 5 + .../buffer/impl/TransactionBufferDisable.java | 5 + .../admin/v3/AdminApiTransactionTest.java | 168 ++++++++++++++++++ .../client/admin/internal/TopicsImpl.java | 10 ++ 8 files changed, 220 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index d56758f75ef2b..5a59fea316670 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -3246,6 +3246,16 @@ private Response generateResponseWithEntry(Entry entry, PersistentTopic persiste if (metadata.hasTxnidMostBits()) { responseBuilder.header("X-Pulsar-txnid-most-bits", metadata.getTxnidMostBits()); } + if (metadata.hasTxnidMostBits() && metadata.hasTxnidLeastBits()) { + TxnID txnID = new TxnID(metadata.getTxnidMostBits(), metadata.getTxnidLeastBits()); + boolean isTxnAborted = persistentTopic.isTxnAborted(txnID, entry.getPosition()); + responseBuilder.header("X-Pulsar-txn-aborted", isTxnAborted); + boolean isTxnUncommitted = persistentTopic.isTxnOngoing(txnID); + responseBuilder.header("X-Pulsar-txn-uncommitted", isTxnUncommitted); + } + boolean isTxnConsumable = entry.getPosition() + .compareTo(persistentTopic.getMaxReadPosition()) <= 0; + responseBuilder.header("X-Pulsar-txn-consumable", isTxnConsumable); if (metadata.hasHighestSequenceId()) { responseBuilder.header("X-Pulsar-highest-sequence-id", metadata.getHighestSequenceId()); } @@ -3264,14 +3274,6 @@ private Response generateResponseWithEntry(Entry entry, PersistentTopic persiste if (metadata.hasNullPartitionKey()) { responseBuilder.header("X-Pulsar-null-partition-key", metadata.isNullPartitionKey()); } - if (metadata.hasTxnidMostBits() && metadata.hasTxnidLeastBits()) { - TxnID txnID = new TxnID(metadata.getTxnidMostBits(), metadata.getTxnidLeastBits()); - boolean isTxnAborted = persistentTopic.isTxnAborted(txnID, entry.getPosition()); - responseBuilder.header("X-Pulsar-txn-aborted", isTxnAborted); - } - boolean isTxnUncommitted = (entry.getPosition()) - .compareTo(persistentTopic.getMaxReadPosition()) > 0; - responseBuilder.header("X-Pulsar-txn-uncommitted", isTxnUncommitted); // Decode if needed CompressionCodec codec = CompressionCodecProvider diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index a882d8a6dbcd8..1117db526b3b8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -4781,6 +4781,10 @@ public boolean isTxnAborted(TxnID txnID, Position readPosition) { return this.transactionBuffer.isTxnAborted(txnID, readPosition); } + public boolean isTxnOngoing(TxnID txnID) { + return this.transactionBuffer.isTxnOngoing(txnID); + } + public TransactionInBufferStats getTransactionInBufferStats(TxnID txnID) { return this.transactionBuffer.getTransactionInBufferStats(txnID); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionBuffer.java index 886f58fdc18b7..6bd7593506da0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionBuffer.java @@ -149,6 +149,14 @@ public interface TransactionBuffer { */ boolean isTxnAborted(TxnID txnID, Position readPosition); + /** + * Check if the transaction is still ongoing (not committed and not aborted). + * + * @param txnID {@link TxnID} the transaction id. + * @return whether the txn is ongoing (uncommitted). + */ + boolean isTxnOngoing(TxnID txnID); + /** * Sync max read position for normal publish. * @param position {@link Position} the position to sync. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBuffer.java index d07370002127a..9164c2bb5fc58 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBuffer.java @@ -380,6 +380,16 @@ public boolean isTxnAborted(TxnID txnID, Position readPosition) { return false; } + @Override + public boolean isTxnOngoing(TxnID txnID) { + TxnBuffer txnBuffer = buffers.get(txnID); + if (txnBuffer == null) { + return false; + } + TxnStatus status = txnBuffer.status(); + return status == TxnStatus.OPEN || status == TxnStatus.COMMITTING || status == TxnStatus.ABORTING; + } + @Override public void syncMaxReadPositionForNormalPublish(Position position, boolean isMarkerMessage) { if (!isMarkerMessage) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 997b455e12f28..e33ac707f1f74 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -682,6 +682,11 @@ public synchronized boolean isTxnAborted(TxnID txnID, Position readPosition) { return snapshotAbortedTxnProcessor.checkAbortedTransaction(txnID); } + @Override + public synchronized boolean isTxnOngoing(TxnID txnID) { + return ongoingTxns.containsKey(txnID); + } + /** * Sync max read position for normal publish. * @param position {@link Position} the position to sync. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferDisable.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferDisable.java index 7d59ab5dd14a2..3718dd8edf0cd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferDisable.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferDisable.java @@ -102,6 +102,11 @@ public boolean isTxnAborted(TxnID txnID, Position readPosition) { return false; } + @Override + public boolean isTxnOngoing(TxnID txnID) { + return false; + } + @Override public void syncMaxReadPositionForNormalPublish(Position position, boolean isMarkerMessage) { if (!isMarkerMessage) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java index 2f1fcf0060505..b3034646293e8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionTest.java @@ -935,6 +935,174 @@ public void testAbortTransaction() throws Exception { } } + @Test + public void testPeekMessageTxnHeaderAccuracy() throws Exception { + initTransaction(1); + + final String topic = BrokerTestUtil.newUniqueName("persistent://public/default/peek_txn_header_accuracy"); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); + + // Start transaction T1, send a message, keep it uncommitted + Transaction txn1 = pulsarClient.newTransaction().build().get(); + TxnID txnID1 = ((TransactionImpl) txn1).getTxnID(); + producer.newMessage(txn1).value("msg-uncommitted").send(); + + // Start transaction T2, send a message, commit it + Transaction txn2 = pulsarClient.newTransaction().build().get(); + TxnID txnID2 = ((TransactionImpl) txn2).getTxnID(); + producer.newMessage(txn2).value("msg-committed").send(); + txn2.commit().get(); + + // Send a normal (non-transactional) message + producer.newMessage().value("msg-normal").send(); + + // Peek all messages with READ_UNCOMMITTED to get both messages regardless of txn state + List> peekMsgs = admin.topics().peekMessages(topic, "t-sub", 10, + false, TransactionIsolationLevel.READ_UNCOMMITTED); + + boolean foundUncommitted = false; + boolean foundCommitted = false; + boolean foundNormal = false; + for (Message peekMsg : peekMsgs) { + String value = new String(peekMsg.getValue()); + MessageMetadata metadata = ((MessageImpl) peekMsg).getMessageBuilder(); + if ("msg-uncommitted".equals(value)) { + foundUncommitted = true; + // T1 is ongoing, so X-Pulsar-txn-uncommitted should be true + assertTrue(peekMsg.hasProperty("X-Pulsar-txn-uncommitted")); + assertEquals(peekMsg.getProperty("X-Pulsar-txn-uncommitted"), "true"); + // T1 is not aborted + assertFalse(peekMsg.hasProperty("X-Pulsar-txn-aborted"), + "T1 is not aborted, X-Pulsar-txn-aborted should be false"); + // Blocked by itself (uncommitted), so X-Pulsar-txn-consumable should be false + assertTrue(peekMsg.hasProperty("X-Pulsar-txn-consumable")); + assertEquals(peekMsg.getProperty("X-Pulsar-txn-consumable"), "false"); + // TxnID checks + assertTrue(metadata.hasTxnidMostBits()); + assertEquals(metadata.getTxnidMostBits(), txnID1.getMostSigBits()); + assertTrue(metadata.hasTxnidLeastBits()); + assertEquals(metadata.getTxnidLeastBits(), txnID1.getLeastSigBits()); + } else if ("msg-committed".equals(value)) { + foundCommitted = true; + // T2 is committed, so X-Pulsar-txn-uncommitted should be false (not present in properties) + assertFalse(peekMsg.hasProperty("X-Pulsar-txn-uncommitted"), + "T2 is committed, X-Pulsar-txn-uncommitted should be false"); + // T2 is not aborted + assertFalse(peekMsg.hasProperty("X-Pulsar-txn-aborted"), + "T2 is not aborted, X-Pulsar-txn-aborted should be false"); + // Blocked by T1 before it, so X-Pulsar-txn-consumable should be false + assertTrue(peekMsg.hasProperty("X-Pulsar-txn-consumable")); + assertEquals(peekMsg.getProperty("X-Pulsar-txn-consumable"), "false", + "T2 is blocked by uncommitted T1 before it"); + // TxnID checks + assertTrue(metadata.hasTxnidMostBits()); + assertEquals(metadata.getTxnidMostBits(), txnID2.getMostSigBits()); + assertTrue(metadata.hasTxnidLeastBits()); + assertEquals(metadata.getTxnidLeastBits(), txnID2.getLeastSigBits()); + } else if ("msg-normal".equals(value)) { + foundNormal = true; + // Normal message has no txnid, so X-Pulsar-txn-uncommitted should not be set + assertFalse(peekMsg.hasProperty("X-Pulsar-txn-uncommitted"), + "Normal message should not have X-Pulsar-txn-uncommitted"); + // Normal message has no txnid, so X-Pulsar-txn-aborted should not be set + assertFalse(peekMsg.hasProperty("X-Pulsar-txn-aborted"), + "Normal message should not have X-Pulsar-txn-aborted"); + // Blocked by T1, so X-Pulsar-txn-consumable should be false + assertTrue(peekMsg.hasProperty("X-Pulsar-txn-consumable")); + assertEquals(peekMsg.getProperty("X-Pulsar-txn-consumable"), "false", + "Normal message is blocked by uncommitted T1"); + // Normal message has no txnid + assertFalse(metadata.hasTxnidMostBits()); + assertFalse(metadata.hasTxnidLeastBits()); + } + } + assertTrue(foundUncommitted, "Should have found the uncommitted message"); + assertTrue(foundCommitted, "Should have found the committed message"); + assertTrue(foundNormal, "Should have found the normal message"); + + // Verify READ_COMMITTED filtering: all messages after an uncommitted txn should be filtered + List> committedPeek = admin.topics().peekMessages(topic, "t-sub-2", 10, + false, TransactionIsolationLevel.READ_COMMITTED); + assertEquals(committedPeek.size(), 0, + "READ_COMMITTED should filter all messages after an uncommitted transaction"); + + // Abort T1 and verify headers update correctly + txn1.abort().get(); + + List> peekAfterAbort = admin.topics().peekMessages(topic, "t-sub-3", 10, + false, TransactionIsolationLevel.READ_UNCOMMITTED); + + boolean foundAborted = false; + boolean foundCommittedAfterAbort = false; + boolean foundNormalAfterAbort = false; + for (Message peekMsg : peekAfterAbort) { + String value = new String(peekMsg.getValue()); + MessageMetadata metadata = ((MessageImpl) peekMsg).getMessageBuilder(); + if ("msg-uncommitted".equals(value)) { + foundAborted = true; + // T1 is now aborted, so X-Pulsar-txn-aborted should be true + assertTrue(peekMsg.hasProperty("X-Pulsar-txn-aborted"), + "T1 is aborted, X-Pulsar-txn-aborted should be true"); + assertEquals(peekMsg.getProperty("X-Pulsar-txn-aborted"), "true"); + // T1 is no longer ongoing + assertFalse(peekMsg.hasProperty("X-Pulsar-txn-uncommitted"), + "T1 is aborted, X-Pulsar-txn-uncommitted should be false"); + // maxReadPosition advanced past T1, so consumable should be true + assertTrue(peekMsg.hasProperty("X-Pulsar-txn-consumable")); + assertEquals(peekMsg.getProperty("X-Pulsar-txn-consumable"), "true", + "T1 is resolved, message should be consumable"); + // TxnID unchanged + assertTrue(metadata.hasTxnidMostBits()); + assertEquals(metadata.getTxnidMostBits(), txnID1.getMostSigBits()); + assertTrue(metadata.hasTxnidLeastBits()); + assertEquals(metadata.getTxnidLeastBits(), txnID1.getLeastSigBits()); + } else if ("msg-committed".equals(value)) { + foundCommittedAfterAbort = true; + // T2 is still committed, not aborted + assertFalse(peekMsg.hasProperty("X-Pulsar-txn-uncommitted"), + "T2 is committed, X-Pulsar-txn-uncommitted should be false"); + assertFalse(peekMsg.hasProperty("X-Pulsar-txn-aborted"), + "T2 is not aborted, X-Pulsar-txn-aborted should be false"); + // maxReadPosition advanced past T1, so consumable should be true + assertTrue(peekMsg.hasProperty("X-Pulsar-txn-consumable")); + assertEquals(peekMsg.getProperty("X-Pulsar-txn-consumable"), "true", + "No open txn before T2, message should be consumable"); + // TxnID unchanged + assertTrue(metadata.hasTxnidMostBits()); + assertEquals(metadata.getTxnidMostBits(), txnID2.getMostSigBits()); + assertTrue(metadata.hasTxnidLeastBits()); + assertEquals(metadata.getTxnidLeastBits(), txnID2.getLeastSigBits()); + } else if ("msg-normal".equals(value)) { + foundNormalAfterAbort = true; + // Normal message has no txnid, so no txn headers + assertFalse(peekMsg.hasProperty("X-Pulsar-txn-uncommitted"), + "Normal message should not have X-Pulsar-txn-uncommitted"); + assertFalse(peekMsg.hasProperty("X-Pulsar-txn-aborted"), + "Normal message should not have X-Pulsar-txn-aborted"); + // No open txn before it, so consumable should be true + assertTrue(peekMsg.hasProperty("X-Pulsar-txn-consumable")); + assertEquals(peekMsg.getProperty("X-Pulsar-txn-consumable"), "true", + "No open txn, normal message should be consumable"); + // No txnid + assertFalse(metadata.hasTxnidMostBits()); + assertFalse(metadata.hasTxnidLeastBits()); + } + } + assertTrue(foundAborted, "Should have found the aborted message after abort"); + assertTrue(foundCommittedAfterAbort, "Should have found the committed message after abort"); + assertTrue(foundNormalAfterAbort, "Should have found the normal message after abort"); + + // READ_COMMITTED should show committed + normal message (aborted is filtered out) + List> committedPeekAfterAbort = admin.topics().peekMessages(topic, "t-sub-4", 10, + false, TransactionIsolationLevel.READ_COMMITTED); + assertEquals(committedPeekAfterAbort.size(), 2, + "READ_COMMITTED should show committed and normal messages after abort"); + assertEquals(new String(committedPeekAfterAbort.get(0).getValue()), "msg-committed"); + assertEquals(new String(committedPeekAfterAbort.get(1).getValue()), "msg-normal"); + } + @Test public void testPeekMessageForSkipTxnMarker() throws Exception { initTransaction(1); diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java index 11dd69a23ce58..4a9c510b5dcb4 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java @@ -133,6 +133,7 @@ public class TopicsImpl extends BaseResource implements Topics { private static final String ENCRYPTION_KEYS = "X-Pulsar-Base64-encryption-keys"; public static final String TXN_ABORTED = "X-Pulsar-txn-aborted"; public static final String TXN_UNCOMMITTED = "X-Pulsar-txn-uncommitted"; + public static final String TXN_CONSUMABLE = "X-Pulsar-txn-consumable"; // CHECKSTYLE.ON: MemberName public static final String PROPERTY_SHADOW_SOURCE_KEY = "PULSAR.SHADOW_SOURCE"; @@ -1330,6 +1331,15 @@ private List> getMessagesFromHttpResponse( } } + tmp = headers.getFirst(TXN_CONSUMABLE); + if (tmp != null) { + properties.put(TXN_CONSUMABLE, tmp.toString()); + if (!Boolean.parseBoolean(tmp.toString()) + && transactionIsolationLevel == TransactionIsolationLevel.READ_COMMITTED) { + return new ArrayList<>(); + } + } + tmp = headers.getFirst(PUBLISH_TIME); if (tmp != null) { messageMetadata.setPublishTime(DateFormatter.parse(tmp.toString())); From ad3e2226b5970e2ccf0dc1d1527bc0f9486e019d Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 10 Aug 2026 15:02:55 +0300 Subject: [PATCH 164/213] [fix][sec][branch-4.2] Upgrade Netty to 4.1.137 to address several CVEs and bugs (#26301) (cherry picked from commit fede2cca5ca0e752ed74e81aa84ba290a4e480ed) --- .../server/src/assemble/LICENSE.bin.txt | 54 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 52 +++++++++--------- pom.xml | 2 +- 3 files changed, 54 insertions(+), 54 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index a0bd04d1d6aa0..09807e47009d1 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -294,33 +294,33 @@ The Apache Software License, Version 2.0 - org.apache.commons-commons-lang3-3.19.0.jar - org.apache.commons-commons-text-1.15.0.jar * Netty - - io.netty-netty-buffer-4.1.136.Final.jar - - io.netty-netty-codec-4.1.136.Final.jar - - io.netty-netty-codec-dns-4.1.136.Final.jar - - io.netty-netty-codec-http-4.1.136.Final.jar - - io.netty-netty-codec-http2-4.1.136.Final.jar - - io.netty-netty-codec-socks-4.1.136.Final.jar - - io.netty-netty-codec-haproxy-4.1.136.Final.jar - - io.netty-netty-common-4.1.136.Final.jar - - io.netty-netty-handler-4.1.136.Final.jar - - io.netty-netty-handler-proxy-4.1.136.Final.jar - - io.netty-netty-resolver-4.1.136.Final.jar - - io.netty-netty-resolver-dns-4.1.136.Final.jar - - io.netty-netty-resolver-dns-classes-macos-4.1.136.Final.jar - - io.netty-netty-resolver-dns-native-macos-4.1.136.Final-osx-aarch_64.jar - - io.netty-netty-resolver-dns-native-macos-4.1.136.Final-osx-x86_64.jar - - io.netty-netty-transport-4.1.136.Final.jar - - io.netty-netty-transport-classes-epoll-4.1.136.Final.jar - - io.netty-netty-transport-native-epoll-4.1.136.Final-linux-aarch_64.jar - - io.netty-netty-transport-native-epoll-4.1.136.Final-linux-x86_64.jar - - io.netty-netty-transport-native-unix-common-4.1.136.Final.jar - - io.netty-netty-tcnative-boringssl-static-2.0.78.Final.jar - - io.netty-netty-tcnative-boringssl-static-2.0.78.Final-linux-aarch_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.78.Final-linux-x86_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.78.Final-osx-aarch_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.78.Final-osx-x86_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.78.Final-windows-x86_64.jar - - io.netty-netty-tcnative-classes-2.0.78.Final.jar + - io.netty-netty-buffer-4.1.137.Final.jar + - io.netty-netty-codec-4.1.137.Final.jar + - io.netty-netty-codec-dns-4.1.137.Final.jar + - io.netty-netty-codec-http-4.1.137.Final.jar + - io.netty-netty-codec-http2-4.1.137.Final.jar + - io.netty-netty-codec-socks-4.1.137.Final.jar + - io.netty-netty-codec-haproxy-4.1.137.Final.jar + - io.netty-netty-common-4.1.137.Final.jar + - io.netty-netty-handler-4.1.137.Final.jar + - io.netty-netty-handler-proxy-4.1.137.Final.jar + - io.netty-netty-resolver-4.1.137.Final.jar + - io.netty-netty-resolver-dns-4.1.137.Final.jar + - io.netty-netty-resolver-dns-classes-macos-4.1.137.Final.jar + - io.netty-netty-resolver-dns-native-macos-4.1.137.Final-osx-aarch_64.jar + - io.netty-netty-resolver-dns-native-macos-4.1.137.Final-osx-x86_64.jar + - io.netty-netty-transport-4.1.137.Final.jar + - io.netty-netty-transport-classes-epoll-4.1.137.Final.jar + - io.netty-netty-transport-native-epoll-4.1.137.Final-linux-aarch_64.jar + - io.netty-netty-transport-native-epoll-4.1.137.Final-linux-x86_64.jar + - io.netty-netty-transport-native-unix-common-4.1.137.Final.jar + - io.netty-netty-tcnative-boringssl-static-2.0.81.Final.jar + - io.netty-netty-tcnative-boringssl-static-2.0.81.Final-linux-aarch_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.81.Final-linux-x86_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.81.Final-osx-aarch_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.81.Final-osx-x86_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.81.Final-windows-x86_64.jar + - io.netty-netty-tcnative-classes-2.0.81.Final.jar - io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.26.Final.jar - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 21ea126a65e3d..964f021994f91 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -345,35 +345,35 @@ The Apache Software License, Version 2.0 - commons-text-1.15.0.jar - commons-compress-1.28.0.jar * Netty - - netty-buffer-4.1.136.Final.jar - - netty-codec-4.1.136.Final.jar - - netty-codec-dns-4.1.136.Final.jar - - netty-codec-http-4.1.136.Final.jar - - netty-codec-socks-4.1.136.Final.jar - - netty-codec-haproxy-4.1.136.Final.jar - - netty-common-4.1.136.Final.jar - - netty-handler-4.1.136.Final.jar - - netty-handler-proxy-4.1.136.Final.jar - - netty-resolver-4.1.136.Final.jar - - netty-resolver-dns-4.1.136.Final.jar - - netty-transport-4.1.136.Final.jar - - netty-transport-classes-epoll-4.1.136.Final.jar - - netty-transport-native-epoll-4.1.136.Final-linux-aarch_64.jar - - netty-transport-native-epoll-4.1.136.Final-linux-x86_64.jar - - netty-transport-native-unix-common-4.1.136.Final.jar - - netty-tcnative-boringssl-static-2.0.78.Final.jar - - netty-tcnative-boringssl-static-2.0.78.Final-linux-aarch_64.jar - - netty-tcnative-boringssl-static-2.0.78.Final-linux-x86_64.jar - - netty-tcnative-boringssl-static-2.0.78.Final-osx-aarch_64.jar - - netty-tcnative-boringssl-static-2.0.78.Final-osx-x86_64.jar - - netty-tcnative-boringssl-static-2.0.78.Final-windows-x86_64.jar - - netty-tcnative-classes-2.0.78.Final.jar + - netty-buffer-4.1.137.Final.jar + - netty-codec-4.1.137.Final.jar + - netty-codec-dns-4.1.137.Final.jar + - netty-codec-http-4.1.137.Final.jar + - netty-codec-socks-4.1.137.Final.jar + - netty-codec-haproxy-4.1.137.Final.jar + - netty-common-4.1.137.Final.jar + - netty-handler-4.1.137.Final.jar + - netty-handler-proxy-4.1.137.Final.jar + - netty-resolver-4.1.137.Final.jar + - netty-resolver-dns-4.1.137.Final.jar + - netty-transport-4.1.137.Final.jar + - netty-transport-classes-epoll-4.1.137.Final.jar + - netty-transport-native-epoll-4.1.137.Final-linux-aarch_64.jar + - netty-transport-native-epoll-4.1.137.Final-linux-x86_64.jar + - netty-transport-native-unix-common-4.1.137.Final.jar + - netty-tcnative-boringssl-static-2.0.81.Final.jar + - netty-tcnative-boringssl-static-2.0.81.Final-linux-aarch_64.jar + - netty-tcnative-boringssl-static-2.0.81.Final-linux-x86_64.jar + - netty-tcnative-boringssl-static-2.0.81.Final-osx-aarch_64.jar + - netty-tcnative-boringssl-static-2.0.81.Final-osx-x86_64.jar + - netty-tcnative-boringssl-static-2.0.81.Final-windows-x86_64.jar + - netty-tcnative-classes-2.0.81.Final.jar - netty-incubator-transport-classes-io_uring-0.0.26.Final.jar - netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar - netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar - - netty-resolver-dns-classes-macos-4.1.136.Final.jar - - netty-resolver-dns-native-macos-4.1.136.Final-osx-aarch_64.jar - - netty-resolver-dns-native-macos-4.1.136.Final-osx-x86_64.jar + - netty-resolver-dns-classes-macos-4.1.137.Final.jar + - netty-resolver-dns-native-macos-4.1.137.Final-osx-aarch_64.jar + - netty-resolver-dns-native-macos-4.1.137.Final-osx-x86_64.jar * Prometheus client - simpleclient-0.16.0.jar - simpleclient_log4j2-0.16.0.jar diff --git a/pom.xml b/pom.xml index bb3867123a952..fd3a5c3f99404 100644 --- a/pom.xml +++ b/pom.xml @@ -187,7 +187,7 @@ flexible messaging model and an intuitive client API. 1.1.10.8 4.1.12.1 5.7.1 - 4.1.136.Final + 4.1.137.Final 0.0.26.Final 12.1.11 From 654d8d07793b62c5a95efa450864508a5c94855f Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 10 Aug 2026 15:38:16 +0300 Subject: [PATCH 165/213] [improve][misc] Upgrade Jetty to 12.1.12 (#26302) (cherry picked from commit 403aae6ea7aa655d73186b4ce4c8c298a5f75f9e) --- .../server/src/assemble/LICENSE.bin.txt | 78 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 24 +++--- pom.xml | 2 +- .../server/FunctionWorkerRoutingTest.java | 16 ++++ 4 files changed, 68 insertions(+), 52 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 09807e47009d1..769e4b1ec184c 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -393,43 +393,43 @@ The Apache Software License, Version 2.0 - org.asynchttpclient-async-http-client-2.15.0.jar - org.asynchttpclient-async-http-client-netty-utils-2.15.0.jar * Jetty - - org.eclipse.jetty-jetty-alpn-client-12.1.11.jar - - org.eclipse.jetty-jetty-alpn-conscrypt-server-12.1.11.jar - - org.eclipse.jetty-jetty-alpn-server-12.1.11.jar - - org.eclipse.jetty-jetty-annotations-12.1.11.jar - - org.eclipse.jetty-jetty-client-12.1.11.jar - - org.eclipse.jetty-jetty-http-12.1.11.jar - - org.eclipse.jetty-jetty-io-12.1.11.jar - - org.eclipse.jetty-jetty-jndi-12.1.11.jar - - org.eclipse.jetty-jetty-plus-12.1.11.jar - - org.eclipse.jetty-jetty-security-12.1.11.jar - - org.eclipse.jetty-jetty-server-12.1.11.jar - - org.eclipse.jetty-jetty-session-12.1.11.jar - - org.eclipse.jetty-jetty-util-12.1.11.jar - - org.eclipse.jetty-jetty-xml-12.1.11.jar - - org.eclipse.jetty.compression-jetty-compression-common-12.1.11.jar - - org.eclipse.jetty.compression-jetty-compression-gzip-12.1.11.jar - - org.eclipse.jetty.compression-jetty-compression-server-12.1.11.jar - - org.eclipse.jetty.ee-jetty-ee-webapp-12.1.11.jar - - org.eclipse.jetty.ee8-jetty-ee8-annotations-12.1.11.jar - - org.eclipse.jetty.ee8-jetty-ee8-nested-12.1.11.jar - - org.eclipse.jetty.ee8-jetty-ee8-plus-12.1.11.jar - - org.eclipse.jetty.ee8-jetty-ee8-proxy-12.1.11.jar - - org.eclipse.jetty.ee8-jetty-ee8-security-12.1.11.jar - - org.eclipse.jetty.ee8-jetty-ee8-servlet-12.1.11.jar - - org.eclipse.jetty.ee8-jetty-ee8-servlets-12.1.11.jar - - org.eclipse.jetty.ee8-jetty-ee8-webapp-12.1.11.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-api-12.1.11.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-common-12.1.11.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-server-12.1.11.jar - - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-servlet-12.1.11.jar + - org.eclipse.jetty-jetty-alpn-client-12.1.12.jar + - org.eclipse.jetty-jetty-alpn-conscrypt-server-12.1.12.jar + - org.eclipse.jetty-jetty-alpn-server-12.1.12.jar + - org.eclipse.jetty-jetty-annotations-12.1.12.jar + - org.eclipse.jetty-jetty-client-12.1.12.jar + - org.eclipse.jetty-jetty-http-12.1.12.jar + - org.eclipse.jetty-jetty-io-12.1.12.jar + - org.eclipse.jetty-jetty-jndi-12.1.12.jar + - org.eclipse.jetty-jetty-plus-12.1.12.jar + - org.eclipse.jetty-jetty-security-12.1.12.jar + - org.eclipse.jetty-jetty-server-12.1.12.jar + - org.eclipse.jetty-jetty-session-12.1.12.jar + - org.eclipse.jetty-jetty-util-12.1.12.jar + - org.eclipse.jetty-jetty-xml-12.1.12.jar + - org.eclipse.jetty.compression-jetty-compression-common-12.1.12.jar + - org.eclipse.jetty.compression-jetty-compression-gzip-12.1.12.jar + - org.eclipse.jetty.compression-jetty-compression-server-12.1.12.jar + - org.eclipse.jetty.ee-jetty-ee-webapp-12.1.12.jar + - org.eclipse.jetty.ee8-jetty-ee8-annotations-12.1.12.jar + - org.eclipse.jetty.ee8-jetty-ee8-nested-12.1.12.jar + - org.eclipse.jetty.ee8-jetty-ee8-plus-12.1.12.jar + - org.eclipse.jetty.ee8-jetty-ee8-proxy-12.1.12.jar + - org.eclipse.jetty.ee8-jetty-ee8-security-12.1.12.jar + - org.eclipse.jetty.ee8-jetty-ee8-servlet-12.1.12.jar + - org.eclipse.jetty.ee8-jetty-ee8-servlets-12.1.12.jar + - org.eclipse.jetty.ee8-jetty-ee8-webapp-12.1.12.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-api-12.1.12.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-common-12.1.12.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-jetty-server-12.1.12.jar + - org.eclipse.jetty.ee8.websocket-jetty-ee8-websocket-servlet-12.1.12.jar - org.eclipse.jetty.toolchain-jetty-servlet-api-4.0.9.jar - - org.eclipse.jetty.websocket-jetty-websocket-core-client-12.1.11.jar - - org.eclipse.jetty.websocket-jetty-websocket-core-common-12.1.11.jar - - org.eclipse.jetty.websocket-jetty-websocket-core-server-12.1.11.jar - - org.eclipse.jetty.websocket-jetty-websocket-jetty-api-12.1.11.jar - - org.eclipse.jetty.websocket-jetty-websocket-jetty-client-12.1.11.jar - - org.eclipse.jetty.websocket-jetty-websocket-jetty-common-12.1.11.jar + - org.eclipse.jetty.websocket-jetty-websocket-core-client-12.1.12.jar + - org.eclipse.jetty.websocket-jetty-websocket-core-common-12.1.12.jar + - org.eclipse.jetty.websocket-jetty-websocket-core-server-12.1.12.jar + - org.eclipse.jetty.websocket-jetty-websocket-jetty-api-12.1.12.jar + - org.eclipse.jetty.websocket-jetty-websocket-jetty-client-12.1.12.jar + - org.eclipse.jetty.websocket-jetty-websocket-jetty-common-12.1.12.jar * SnakeYaml -- org.yaml-snakeyaml-2.0.jar * RocksDB - org.rocksdb-rocksdbjni-7.9.2.jar * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.45.0.jar @@ -567,9 +567,9 @@ BSD 3-clause "New" or "Revised" License * JSR305 -- com.google.code.findbugs-jsr305-3.0.2.jar -- ../licenses/LICENSE-JSR305.txt * JLine3 -- org.jline-jline-4.2.1.jar -- ../licenses/LICENSE-JLine.txt * OW2 ASM - - org.ow2.asm-asm-9.10.jar -- ../licenses/LICENSE-ASM.txt - - org.ow2.asm-asm-commons-9.10.jar -- ../licenses/LICENSE-ASM.txt - - org.ow2.asm-asm-tree-9.10.jar -- ../licenses/LICENSE-ASM.txt + - org.ow2.asm-asm-9.10.1.jar -- ../licenses/LICENSE-ASM.txt + - org.ow2.asm-asm-commons-9.10.1.jar -- ../licenses/LICENSE-ASM.txt + - org.ow2.asm-asm-tree-9.10.1.jar -- ../licenses/LICENSE-ASM.txt BSD 2-Clause License * HdrHistogram -- org.hdrhistogram-HdrHistogram-2.1.9.jar -- ../licenses/LICENSE-HdrHistogram.txt diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 964f021994f91..d7796bfb49036 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -401,18 +401,18 @@ The Apache Software License, Version 2.0 - async-http-client-2.15.0.jar - async-http-client-netty-utils-2.15.0.jar * Jetty - - jetty-alpn-client-12.1.11.jar - - jetty-client-12.1.11.jar - - jetty-compression-common-12.1.11.jar - - jetty-compression-gzip-12.1.11.jar - - jetty-http-12.1.11.jar - - jetty-io-12.1.11.jar - - jetty-util-12.1.11.jar - - jetty-websocket-core-client-12.1.11.jar - - jetty-websocket-core-common-12.1.11.jar - - jetty-websocket-jetty-api-12.1.11.jar - - jetty-websocket-jetty-client-12.1.11.jar - - jetty-websocket-jetty-common-12.1.11.jar + - jetty-alpn-client-12.1.12.jar + - jetty-client-12.1.12.jar + - jetty-compression-common-12.1.12.jar + - jetty-compression-gzip-12.1.12.jar + - jetty-http-12.1.12.jar + - jetty-io-12.1.12.jar + - jetty-util-12.1.12.jar + - jetty-websocket-core-client-12.1.12.jar + - jetty-websocket-core-common-12.1.12.jar + - jetty-websocket-jetty-api-12.1.12.jar + - jetty-websocket-jetty-client-12.1.12.jar + - jetty-websocket-jetty-common-12.1.12.jar * SnakeYaml -- snakeyaml-2.0.jar * Google Error Prone Annotations - error_prone_annotations-2.45.0.jar * Javassist -- javassist-3.25.0-GA.jar diff --git a/pom.xml b/pom.xml index fd3a5c3f99404..3b876fc487bca 100644 --- a/pom.xml +++ b/pom.xml @@ -189,7 +189,7 @@ flexible messaging model and an intuitive client API. 5.7.1 4.1.137.Final 0.0.26.Final - 12.1.11 + 12.1.12 9.4.58.v20250814 2.5.2 diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/FunctionWorkerRoutingTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/FunctionWorkerRoutingTest.java index 1188fc164cd2b..499a44ae2d33e 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/FunctionWorkerRoutingTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/FunctionWorkerRoutingTest.java @@ -20,6 +20,8 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import javax.servlet.ServletConfig; +import javax.servlet.ServletContext; import javax.servlet.http.HttpServletRequest; import lombok.Cleanup; import org.apache.pulsar.client.api.Authentication; @@ -45,7 +47,11 @@ public void testFunctionWorkerRedirect() throws Exception { proxyClientAuthentication.start(); BrokerDiscoveryProvider discoveryProvider = mock(BrokerDiscoveryProvider.class); + @Cleanup("destroy") AdminProxyHandler handler = new AdminProxyHandler(proxyConfig, discoveryProvider, proxyClientAuthentication); + // rewriteTarget() delegates to Jetty's AbstractProxyServlet, which relies on state that the servlet + // container sets up, so initialize the servlet the same way ProxyServiceStarter's ServletHolder does. + handler.init(buildServletConfig()); String funcUrl = handler.rewriteTarget(buildRequest("/admin/v3/functions/test/test")); Assert.assertEquals(funcUrl, String.format("%s/admin/v3/functions/%s/%s", @@ -64,6 +70,16 @@ public void testFunctionWorkerRedirect() throws Exception { brokerUrl, "test")); } + static ServletConfig buildServletConfig() { + ServletConfig servletConfig = mock(ServletConfig.class); + when(servletConfig.getServletName()).thenReturn("admin-proxy"); + when(servletConfig.getServletContext()).thenReturn(mock(ServletContext.class)); + // outside of a running Jetty server there is no server executor to borrow, so let the proxy servlet + // create a thread pool of its own + when(servletConfig.getInitParameter("maxThreads")).thenReturn("8"); + return servletConfig; + } + static HttpServletRequest buildRequest(String url) { HttpServletRequest mockReq = mock(HttpServletRequest.class); when(mockReq.getRequestURI()).thenReturn(url); From 4b5704b7a59da37d4945cd35f575dd63db2f1a8b Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Mon, 10 Aug 2026 23:55:59 +0800 Subject: [PATCH 166/213] [fix][ml] Skip contiguous deleted ranges during reads (#26299) (cherry picked from commit a87e36db719a8411bc987b867742933624b43a0c) --- .../mledger/impl/ManagedLedgerImpl.java | 9 +- .../ManagedCursorSkipDeletedEntriesTest.java | 142 ++++++++++++++++++ 2 files changed, 148 insertions(+), 3 deletions(-) create mode 100644 managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorSkipDeletedEntriesTest.java diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 52f1201e32cc2..a14ce1944d755 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -2455,10 +2455,13 @@ private void internalReadFromLedger(ReadHandle ledger, OpReadEntry opReadEntry) } } - // If all messages in [firstEntry...lastEntry] are filter out, - // then manual call internalReadEntriesComplete to advance read position. + // If all positions in [firstEntry...lastEntry] are filtered out, advance the read position + // without issuing a ledger read. if (firstValidEntry == -1L) { - final var nextReadPosition = PositionFactory.create(ledger.getId(), lastEntry).getNext(); + final Position lastScannedPosition = PositionFactory.create(ledger.getId(), lastEntry); + // The whole scan window was skipped. If the last scanned position is individually deleted, + // hop over its deleted range instead of advancing one window at a time. + final Position nextReadPosition = opReadEntry.cursor.getNextAvailablePosition(lastScannedPosition); opReadEntry.updateReadPosition(nextReadPosition); opReadEntry.checkReadCompletion(); return; diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorSkipDeletedEntriesTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorSkipDeletedEntriesTest.java new file mode 100644 index 0000000000000..8c7dbd93dbf2d --- /dev/null +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorSkipDeletedEntriesTest.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.impl; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Predicate; +import lombok.Cleanup; +import org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntriesCallback; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.ManagedLedgerConfig; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.bookkeeper.test.MockedBookKeeperTestCase; +import org.testng.annotations.Test; + +/** + * Regression test for reading over a cursor that has a very large contiguous range of individually + * deleted (acknowledged) entries while its mark-delete position is pinned by an older unacknowledged entry. + * + *

This is the shape a Key_Shared subscription ends up in when a single unacknowledged message blocks the + * mark-delete position while consumers keep acknowledging everything after it. Reads then have to get + * past tens of millions of already-acknowledged entries before they can fill a batch. + * + *

The skip pre-filter in {@link ManagedLedgerImpl#internalReadFromLedger} used to advance the read + * position by the size of the scan window when every entry in that window was already acknowledged, instead of + * hopping over the whole deleted range via + * {@link ManagedCursorImpl#getNextAvailablePosition(Position)}. That turned a single range hop into + * O(entries / batchSize) read-loop iterations. Each iteration re-ran + * {@code isLedgerFullyAcked} -> {@code ManagedCursorImpl#getNumberOfEntries} -> + * {@code RangeSetWrapper#cardinality}, which clones the cursor's per-ledger Roaring bitmap. + */ +public class ManagedCursorSkipDeletedEntriesTest extends MockedBookKeeperTestCase { + + private static final int TOTAL_ENTRIES = 20_000; + private static final int READ_BATCH_SIZE = 100; + /** Entries {@code [1, LAST_DELETED_INDEX]} are individually deleted, forming one contiguous range. */ + private static final int LAST_DELETED_INDEX = TOTAL_ENTRIES - 4; + + /** + * A read that has to get past one large contiguous deleted range must hop over it rather than walk + * it one batch at a time. + */ + @Test(timeOut = 300_000) + public void testReadHopsOverLargeDeletedRangeInsteadOfWalkingIt() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig() + // Keep every entry below in a single ledger, then force a rollover so that the reads + // under test target a closed ledger. That makes every read-loop iteration go through + // the isLedgerFullyAcked check, as it does in production. + .setMaxEntriesPerLedger(TOTAL_ENTRIES) + .setRetentionTime(1, TimeUnit.HOURS) + .setRetentionSizeInMB(-1); + + @Cleanup + ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("skip-deleted-entries", config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("sub"); + + List positions = new ArrayList<>(TOTAL_ENTRIES); + for (int i = 0; i < TOTAL_ENTRIES; i++) { + positions.add(ledger.addEntry(new byte[]{1})); + } + // Roll the ledger holding the entries above, so it is closed by the time it is read. + Position afterRollover = ledger.addEntry(new byte[]{1}); + assertTrue(ledger.getLedgersInfoAsList().size() >= 2, + "expected the entries under test to live in a closed ledger, ledgers=" + + ledger.getLedgersInfoAsList().size()); + + // Individually acknowledge [1, LAST_DELETED_INDEX]. Entry 0 stays unacknowledged and pins the + // mark-delete position, so none of these acknowledgments can be collapsed into it. + cursor.delete(positions.subList(1, LAST_DELETED_INDEX + 1)); + + assertEquals(cursor.getMarkDeletedPosition(), PositionFactory.create(positions.get(0).getLedgerId(), -1), + "mark-delete position must stay pinned behind the unacknowledged entry 0"); + assertEquals(cursor.getTotalNonContiguousDeletedMessagesRange(), 1, + "the acknowledgments must collapse into a single contiguous range"); + + // The deliverable entries are entry 0, the tail holes, and the entry that caused the rollover. + List expected = new ArrayList<>(); + expected.add(positions.get(0)); + for (int i = LAST_DELETED_INDEX + 1; i < TOTAL_ENTRIES; i++) { + expected.add(positions.get(i)); + } + expected.add(afterRollover); + + // Count every position the skip pre-filter examines. Predicate#or evaluates this one first, so + // it sees exactly the positions ManagedLedgerImpl walks before deciding what to read. + AtomicInteger examinedPositions = new AtomicInteger(); + Predicate countingCondition = position -> { + examinedPositions.incrementAndGet(); + return false; + }; + + CompletableFuture> readFuture = new CompletableFuture<>(); + cursor.asyncReadEntriesWithSkip(READ_BATCH_SIZE, -1L, new ReadEntriesCallback() { + @Override + public void readEntriesComplete(List entries, Object ctx) { + readFuture.complete(entries); + } + + @Override + public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + readFuture.completeExceptionally(exception); + } + }, null, PositionFactory.LATEST, countingCondition); + + List read = readFuture.get(120, TimeUnit.SECONDS); + List readPositions = new ArrayList<>(); + for (Entry entry : read) { + readPositions.add(entry.getPosition()); + entry.release(); + } + assertEquals(readPositions, expected, "the read must return exactly the unacknowledged entries"); + + // Hopping the range examines one scan window before jumping it and a small number of windows for + // the entries actually returned. Walking it examines every position in the deleted block. + assertTrue(examinedPositions.get() <= 4 * READ_BATCH_SIZE, + "read examined " + examinedPositions.get() + " positions to return " + read.size() + + " entries over a deleted range of " + LAST_DELETED_INDEX + " entries"); + } +} From 9175b4ed5d8ec996df6e2063b70a191342cb7b5b Mon Sep 17 00:00:00 2001 From: Yong Zhang Date: Tue, 11 Aug 2026 15:37:56 +0800 Subject: [PATCH 167/213] [improve][broker] Expose interface for the replicator in ManagedLedger instead of cast the class (#26298) (cherry picked from commit 34124275133f411e7eb109a6131feb19771ff43b) --- .../bookkeeper/mledger/ManagedCursor.java | 16 ++++++++++ .../mledger/impl/ManagedCursorImpl.java | 7 +++++ .../mledger/impl/ManagedCursorTest.java | 31 +++++++++++++++++++ .../persistent/PersistentReplicator.java | 8 +---- 4 files changed, 55 insertions(+), 7 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java index 4e5e12365480c..340a5c4306287 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java @@ -18,12 +18,14 @@ */ package org.apache.bookkeeper.mledger; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.google.common.collect.Range; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import java.util.function.Predicate; import org.apache.bookkeeper.common.annotation.InterfaceAudience; import org.apache.bookkeeper.common.annotation.InterfaceStability; @@ -860,6 +862,20 @@ default void skipNonRecoverableLedger(long ledgerId){} */ ManagedLedger getManagedLedger(); + /** + * Schedule a continuation of a read callback. + * + *

Implementations that deliver read callbacks on a dedicated execution context should override this method + * to run the continuation on that same execution context. + * + * @param callback the callback continuation + * @param delay the delay before executing the continuation + * @param unit the time unit of the delay + */ + default void scheduleReadCallback(Runnable callback, long delay, TimeUnit unit) { + CompletableFuture.delayedExecutor(delay, unit).execute(catchingAndLoggingThrowables(callback)); + } + /** * Get last individual deleted range. * @return range diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 7a2ceecbd3d07..49cec68322666 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -26,6 +26,7 @@ import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.DEFAULT_LEDGER_DELETE_RETRIES; import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.createManagedLedgerException; import static org.apache.bookkeeper.mledger.util.Errors.isNoSuchLedgerExistsException; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; import com.google.common.collect.Collections2; @@ -4003,6 +4004,12 @@ public ManagedLedger getManagedLedger() { return this.ledger; } + @Override + public void scheduleReadCallback(Runnable callback, long delay, TimeUnit unit) { + ledger.getScheduledExecutor().schedule( + catchingAndLoggingThrowables(() -> ledger.getExecutor().execute(callback)), delay, unit); + } + @Override public Range getLastIndividualDeletedRange() { lock.readLock().lock(); diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java index 8f13c41d407b4..1dcff5762802a 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java @@ -96,6 +96,7 @@ import org.apache.bookkeeper.client.api.LedgerEntries; import org.apache.bookkeeper.client.api.ReadHandle; import org.apache.bookkeeper.common.util.OrderedExecutor; +import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCallback; @@ -134,6 +135,7 @@ import org.apache.pulsar.metadata.api.extended.SessionEvent; import org.apache.pulsar.metadata.impl.FaultInjectionMetadataStore; import org.awaitility.Awaitility; +import org.mockito.ArgumentCaptor; import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; @@ -3818,6 +3820,35 @@ public void testEstimatedUnackedSizeWhenCursorCaughtUpWithLastPosition() throws assertEquals(cursor.getEstimatedSizeSinceMarkDeletePosition(), 0); } + @Test + public void testScheduleReadCallbackUsesManagedLedgerExecutionContext() { + ManagedLedgerImpl ledger = mock(ManagedLedgerImpl.class); + when(ledger.getConfig()).thenReturn(new ManagedLedgerConfig()); + OrderedScheduler scheduledExecutor = mock(OrderedScheduler.class); + ExecutorService executor = mock(ExecutorService.class); + when(ledger.getScheduledExecutor()).thenReturn(scheduledExecutor); + when(ledger.getExecutor()).thenReturn(executor); + ManagedCursorImpl cursor = new ManagedCursorImpl(mock(BookKeeper.class), ledger, "c1"); + Runnable callback = mock(Runnable.class); + ArgumentCaptor scheduledTask = ArgumentCaptor.forClass(Runnable.class); + + cursor.scheduleReadCallback(callback, 100, TimeUnit.MILLISECONDS); + + verify(scheduledExecutor).schedule(scheduledTask.capture(), eq(100L), eq(TimeUnit.MILLISECONDS)); + scheduledTask.getValue().run(); + verify(executor).execute(callback); + } + + @Test + public void testDefaultScheduleReadCallback() throws InterruptedException { + ManagedCursor cursor = mock(ManagedCursor.class, Mockito.CALLS_REAL_METHODS); + CountDownLatch callbackExecuted = new CountDownLatch(1); + + cursor.scheduleReadCallback(callbackExecuted::countDown, 0, TimeUnit.MILLISECONDS); + + assertTrue(callbackExecuted.await(5, TimeUnit.SECONDS)); + } + @Test public void testEstimatedUnackedSizeWhenCursorAdvancedToEmptyCurrentLedger() { ManagedLedgerImpl ledger = mock(ManagedLedgerImpl.class); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java index c1411592ddd07..b51508afab5af 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java @@ -52,7 +52,6 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.CursorAlreadyClosedException; import org.apache.bookkeeper.mledger.ManagedLedgerException.TooManyRequestsException; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.PulsarServerException; @@ -434,7 +433,6 @@ public void readEntriesComplete(List entries, Object ctx) { } // Retry to trigger read completes if it is not started. - ManagedLedgerImpl ml = (ManagedLedgerImpl) cursor.getManagedLedger(); Runnable retryReplicateEntries = () -> { long estimatedTimeStampProducerConnected = this.estimatedTimeStampProducerConnected; long delayMillis; @@ -443,11 +441,7 @@ public void readEntriesComplete(List entries, Object ctx) { } else { delayMillis = 100; } - ml.getScheduledExecutor().schedule(() -> { - ml.getExecutor().execute(() -> { - readEntriesComplete(entries, ctx); - }); - }, delayMillis, TimeUnit.MILLISECONDS); + cursor.scheduleReadCallback(() -> readEntriesComplete(entries, ctx), delayMillis, TimeUnit.MILLISECONDS); }; // Retry. From a2c8d1788cb27b05331254047d7c20a4c07d3825 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Wed, 12 Aug 2026 11:38:42 +0800 Subject: [PATCH 168/213] [Fix][broker]Get a fenced error when loading a topic that does not allowed the cluster to access (#26276) (cherry picked from commit 2babfbcc6b7beca22fec0b9da887c28e28e8ca7c) --- .../pulsar/broker/service/BrokerService.java | 2 +- .../service/persistent/PersistentTopic.java | 44 +++++++++++++++++++ ...yReplicatorUsingGlobalPartitionedTest.java | 12 +++++ .../PersistentTopicInitializeDelayTest.java | 2 + 4 files changed, 59 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 30b2f439d603e..da4a812e6cc97 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -1986,7 +1986,7 @@ public void openLedgerComplete(ManagedLedger ledger, Object ctx) { .thenCompose(__ -> context.trace("pre-create compacted sub", persistentTopic.preCreateSubscriptionForCompactionIfNeeded())) .thenCompose(__ -> context.trace("replication", - persistentTopic.checkReplication())) + persistentTopic.initializeCheckReplication())) .thenCompose(v -> context.trace("deduplication", persistentTopic.checkDeduplicationStatus())) .thenRun(() -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 1117db526b3b8..aebec1d0d31bb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -342,6 +342,9 @@ private static class EstimateTimeBasedBacklogQuotaCheckResult { // For active topics, the ledger provides the timestamp directly, so this cache is cleared. private volatile long cachedLastPublishTimestamp; + private final CompletableFuture initialReplicationCheck = new CompletableFuture<>(); + private final AtomicBoolean initialReplicationCheckInitialized = new AtomicBoolean(false); + /*** * We use 3 futures to prevent a new closing if there is an in-progress deletion or closing. We make Pulsar return * the in-progress one when it is called the second time. @@ -1971,8 +1974,49 @@ CompletableFuture checkPersistencePolicies() { return future; } + /** + * Starts the replication check used while loading a topic. + * + *

The initialization state is tracked separately from regular replication checks so that a concurrent + * policy update does not start a second initial check. The check itself is still dispatched through + * {@link #checkReplication()} to preserve overrides supplied by + * {@link org.apache.pulsar.broker.service.TopicFactory}. + */ + public final CompletableFuture initializeCheckReplication() { + if (initialReplicationCheckInitialized.compareAndSet(false, true)) { + try { + checkReplication().whenComplete((__, ex) -> { + if (ex != null) { + initialReplicationCheck.completeExceptionally(ex); + } else { + initialReplicationCheck.complete(null); + } + }); + } catch (Throwable t) { + // checkReplication is overridable, so an implementation can fail before returning its future. + initialReplicationCheck.completeExceptionally(t); + } + } + // Do not expose the mutable internal future: callers can cancel or complete a CompletableFuture. + return initialReplicationCheck.thenApply(__ -> null); + } + @Override public CompletableFuture checkReplication() { + if (initialReplicationCheckInitialized.compareAndSet(false, true)) { + internalCheckReplication().whenComplete((__, ex) -> { + if (ex != null) { + initialReplicationCheck.completeExceptionally(ex); + } else { + initialReplicationCheck.complete(null); + } + }); + return initialReplicationCheck.thenApply(__ -> null); + } + return internalCheckReplication(); + } + + private CompletableFuture internalCheckReplication() { TopicName name = TopicName.get(topic); if (!name.isGlobal() || NamespaceService.isHeartbeatNamespace(name) || ExtensibleLoadManagerImpl.isInternalTopic(topic)) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalPartitionedTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalPartitionedTest.java index cdeeaba29506d..0d34479068b40 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalPartitionedTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorUsingGlobalPartitionedTest.java @@ -23,6 +23,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import java.time.Duration; import java.util.Arrays; import java.util.HashSet; @@ -334,6 +335,17 @@ public void testRemoveCluster(String removeClusterLevel) throws Exception { "Remote cluster should have local policies: publish rate."); }); + CompletableFuture> future = pulsar1.getBrokerService().getTopic(topicP1, true); + if ("topic".equals(removeClusterLevel)) { + future.get(90, TimeUnit.SECONDS); + } else { + try { + future.get(90, TimeUnit.SECONDS); + fail("Should have thrown an exception since the __change_event topic can not be access anymore"); + } catch (Exception e) { + assertTrue(e.getMessage().contains("Namespace missing local cluster name")); + } + } // cleanup. if ("topic".equals(removeClusterLevel)) { admin1.namespaces().setNamespaceReplicationClusters(ns1, new HashSet<>(Arrays.asList(cluster2))); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicInitializeDelayTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicInitializeDelayTest.java index f528032299aee..7968fd45cdbb4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicInitializeDelayTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicInitializeDelayTest.java @@ -72,10 +72,12 @@ public void testTopicInitializeDelay() throws Exception { admin.topicPolicies().setMaxConsumers(topicName, 10); Awaitility.await().untilAsserted(() -> assertEquals(admin.topicPolicies().getMaxConsumers(topicName), 10)); admin.topics().unload(topicName); + MyPersistentTopic.checkReplicationInvocationCount.set(0); CompletableFuture> optionalFuture = pulsar.getBrokerService().getTopic(topicName, true); Optional topic = optionalFuture.get(15, TimeUnit.SECONDS); assertTrue(topic.isPresent()); + assertEquals(MyPersistentTopic.checkReplicationInvocationCount.get(), 1); } public static class MyTopicFactory implements TopicFactory { From 0d10a1bd0d4b766250278049546acd00c9e443ef Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 12 Aug 2026 16:33:37 +0300 Subject: [PATCH 169/213] [improve][misc] Upgrade Conscrypt to 2.6.1 to add aarch64 native support (#26314) (cherry picked from commit 86ff200aa59313c4c1b0f0c7b7619b7b7a562c5b) --- .../server/src/assemble/LICENSE.bin.txt | 2 +- .../shell/src/assemble/LICENSE.bin.txt | 2 +- pom.xml | 2 +- .../pulsar/common/util/SecurityUtility.java | 54 ++----------------- .../util/keystoretls/KeyStoreSSLContext.java | 2 +- 5 files changed, 9 insertions(+), 53 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 769e4b1ec184c..2519a0dce53cc 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -261,7 +261,7 @@ The Apache Software License, Version 2.0 - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.18.9.jar - com.fasterxml.jackson.module-jackson-module-parameter-names-2.18.9.jar * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.9.1.jar - * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar + * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.6.1.jar * Fastutil -- it.unimi.dsi-fastutil-8.5.16.jar * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.59.2.jar * Bitbucket -- org.bitbucket.b_c-jose4j-0.9.6.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index d7796bfb49036..1e6cbef8c2367 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -325,7 +325,7 @@ The Apache Software License, Version 2.0 - jackson-datatype-jsr310-2.18.9.jar - jackson-module-parameter-names-2.18.9.jar * Caffeine -- caffeine-2.9.1.jar - * Conscrypt -- conscrypt-openjdk-uber-2.5.2.jar + * Conscrypt -- conscrypt-openjdk-uber-2.6.1.jar * Gson - gson-2.13.2.jar * Guava diff --git a/pom.xml b/pom.xml index 3b876fc487bca..02b7ae9a52c1b 100644 --- a/pom.xml +++ b/pom.xml @@ -192,7 +192,7 @@ flexible messaging model and an intuitive client API. 12.1.12 9.4.58.v20250814 - 2.5.2 + 2.6.1 2.42 1.10.62 0.16.0 diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java index 80c692c9d6f01..01307d7e5e021 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SecurityUtility.java @@ -69,7 +69,6 @@ import javax.net.ssl.TrustManagerFactory; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.common.classification.InterfaceAudience; import org.apache.pulsar.common.tls.TlsHostnameVerifier; /** @@ -162,10 +161,11 @@ private static Provider loadConscryptProvider() { // // more details of Conscrypt's hostname verification: // https://github.com/google/conscrypt/blob/master/IMPLEMENTATION_NOTES.md#hostname-verification - // there's a bug in Conscrypt while setting a custom HostnameVerifier, - // https://github.com/google/conscrypt/issues/1015 and therefore this solution alone - // isn't sufficient to configure Conscrypt's hostname verifier. The method processConscryptTrustManager - // contains the workaround. + // + // Setting the default is sufficient on its own since Conscrypt 2.6.0: TrustManagerImpl used to ignore + // the default verifier (https://github.com/google/conscrypt/issues/1015), which forced Pulsar to copy it + // onto every TrustManager instance, but https://github.com/google/conscrypt/pull/1060 made it fall back + // to the default, so that workaround has been removed. try { HostnameVerifier hostnameVerifier = new TlsHostnameVerifier(); Object wrappedHostnameVerifier = conscryptClazz @@ -397,54 +397,10 @@ private static TrustManager[] setupTrustCerts(KeyStoreHolder ksh, boolean allowI } trustManagers = tmf.getTrustManagers(); - - for (TrustManager trustManager : trustManagers) { - processConscryptTrustManager(trustManager); - } } return trustManagers; } - /*** - * Conscrypt TrustManager instances will be configured to use the Pulsar {@link TlsHostnameVerifier} - * class. - * This method is used as a workaround for https://github.com/google/conscrypt/issues/1015 - * when Conscrypt / OpenSSL is used as the TLS security provider. - * - * @param trustManagers the array of TrustManager instances to process. - * @return same instance passed as parameter - */ - @InterfaceAudience.Private - public static TrustManager[] processConscryptTrustManagers(TrustManager[] trustManagers) { - for (TrustManager trustManager : trustManagers) { - processConscryptTrustManager(trustManager); - } - return trustManagers; - } - - // workaround https://github.com/google/conscrypt/issues/1015 - private static void processConscryptTrustManager(TrustManager trustManager) { - if (trustManager.getClass().getName().equals("org.conscrypt.TrustManagerImpl")) { - try { - Class conscryptClazz = Class.forName("org.conscrypt.Conscrypt"); - Object hostnameVerifier = conscryptClazz.getMethod("getHostnameVerifier", - new Class[]{TrustManager.class}).invoke(null, trustManager); - if (hostnameVerifier == null) { - Object defaultHostnameVerifier = conscryptClazz.getMethod("getDefaultHostnameVerifier", - new Class[]{TrustManager.class}).invoke(null, trustManager); - if (defaultHostnameVerifier != null) { - conscryptClazz.getMethod("setHostnameVerifier", new Class[]{ - TrustManager.class, - Class.forName("org.conscrypt.ConscryptHostnameVerifier") - }).invoke(null, trustManager, defaultHostnameVerifier); - } - } - } catch (ReflectiveOperationException e) { - log.warn("Unable to set hostname verifier for Conscrypt TrustManager implementation", e); - } - } - } - public static X509Certificate[] loadCertificatesFromPemFile(String certFilePath) throws KeyManagementException { X509Certificate[] certificates = null; diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/KeyStoreSSLContext.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/KeyStoreSSLContext.java index a70857bdf3b5f..f8d05ee9afce9 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/KeyStoreSSLContext.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/KeyStoreSSLContext.java @@ -170,7 +170,7 @@ public SSLContext createSSLContext() throws GeneralSecurityException, IOExceptio TrustManager[] trustManagers = null; if (trustManagerFactory != null) { - trustManagers = SecurityUtility.processConscryptTrustManagers(trustManagerFactory.getTrustManagers()); + trustManagers = trustManagerFactory.getTrustManagers(); } // init From 0a64cb7b39fd21c627b62d9e0168e880936d6edf Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Thu, 13 Aug 2026 17:16:48 +0800 Subject: [PATCH 170/213] [fix][client] Restore OAuth2 HTTP client after deserialization (#26325) --- .../auth/oauth2/ClientCredentialsFlow.java | 2 +- .../client/impl/auth/oauth2/FlowBase.java | 28 +++++++++++++--- .../impl/auth/oauth2/TlsClientAuthFlow.java | 4 +-- .../auth/oauth2/AuthenticationOAuth2Test.java | 32 ++++++++++++++++++- 4 files changed, 57 insertions(+), 9 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/ClientCredentialsFlow.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/ClientCredentialsFlow.java index 6692e86848b73..95f7bc56cb85e 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/ClientCredentialsFlow.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/ClientCredentialsFlow.java @@ -147,7 +147,7 @@ public void initialize() throws PulsarClientException { assert this.metadata != null; URL tokenUrl = this.metadata.getTokenEndpoint(); - this.exchanger = new TokenClient(tokenUrl, httpClient); + this.exchanger = new TokenClient(tokenUrl, getHttpClient()); initialized = true; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/FlowBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/FlowBase.java index d1caa82e07c22..b6d7b39f236ff 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/FlowBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/FlowBase.java @@ -67,21 +67,31 @@ abstract class FlowBase implements Flow { private static final long serialVersionUID = 1L; protected final URL issuerUrl; - protected transient AsyncHttpClient httpClient; + private final Duration connectTimeout; + private final Duration readTimeout; + private final String trustCertsFilePath; + private final String certFile; + private final String keyFile; + private final long autoCertRefreshSeconds; protected final String wellKnownMetadataPath; protected transient PulsarSslFactory sslFactory; protected transient ScheduledExecutorService sslRefreshScheduler; protected transient Metadata metadata; + private transient AsyncHttpClient httpClient; protected FlowBase(URL issuerUrl, Duration connectTimeout, Duration readTimeout, String trustCertsFilePath, String certFile, String keyFile, Duration autoCertRefreshDuration, String wellKnownMetadataPath) { this.issuerUrl = issuerUrl; - this.httpClient = defaultHttpClient(readTimeout, connectTimeout, trustCertsFilePath, certFile, keyFile); - long autoCertRefreshSeconds = getParameterDurationToSeconds(CONFIG_PARAM_AUTO_CERT_REFRESH_DURATION, + this.connectTimeout = connectTimeout; + this.readTimeout = readTimeout; + this.trustCertsFilePath = trustCertsFilePath; + this.certFile = certFile; + this.keyFile = keyFile; + this.autoCertRefreshSeconds = getParameterDurationToSeconds(CONFIG_PARAM_AUTO_CERT_REFRESH_DURATION, autoCertRefreshDuration, DEFAULT_AUTO_CERT_REFRESH_DURATION); - scheduleSslContextRefreshIfEnabled(autoCertRefreshSeconds); this.wellKnownMetadataPath = wellKnownMetadataPath; + getHttpClient(); } private AsyncHttpClient defaultHttpClient(Duration readTimeout, Duration connectTimeout, @@ -132,6 +142,14 @@ private AsyncHttpClient defaultHttpClient(Duration readTimeout, Duration connect return new DefaultAsyncHttpClient(confBuilder.build()); } + protected synchronized AsyncHttpClient getHttpClient() { + if (httpClient == null) { + httpClient = defaultHttpClient(readTimeout, connectTimeout, trustCertsFilePath, certFile, keyFile); + scheduleSslContextRefreshIfEnabled(autoCertRefreshSeconds); + } + return httpClient; + } + private void scheduleSslContextRefreshIfEnabled(long refreshSeconds) { if (sslFactory == null || refreshSeconds <= 0 || sslRefreshScheduler != null) { return; @@ -185,7 +203,7 @@ public void initialize() throws PulsarClientException { } protected MetadataResolver createMetadataResolver() { - return DefaultMetadataResolver.fromIssuerUrl(issuerUrl, httpClient, wellKnownMetadataPath); + return DefaultMetadataResolver.fromIssuerUrl(issuerUrl, getHttpClient(), wellKnownMetadataPath); } static String parseParameterString(Map params, String name) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/TlsClientAuthFlow.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/TlsClientAuthFlow.java index d61dadef83bfb..d592002b4a458 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/TlsClientAuthFlow.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/auth/oauth2/TlsClientAuthFlow.java @@ -113,7 +113,7 @@ public void initialize() throws PulsarClientException { assert this.metadata != null; URL tokenUrl = this.metadata.getTokenEndpoint(); - this.exchanger = new TokenClient(tokenUrl, httpClient); + this.exchanger = new TokenClient(tokenUrl, getHttpClient()); initialized = true; } @@ -152,4 +152,4 @@ public void close() throws Exception { String getClientId() { return clientId; } -} \ No newline at end of file +} diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationOAuth2Test.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationOAuth2Test.java index 6b956957d307b..e87f3f5eb86cc 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationOAuth2Test.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/auth/oauth2/AuthenticationOAuth2Test.java @@ -28,6 +28,10 @@ import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertThrows; import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.net.MalformedURLException; import java.net.URI; import java.net.URL; @@ -119,6 +123,32 @@ public void testConfigureWithoutOptionalParams() throws Exception { assertNotNull(this.auth.flow); } + @Test + public void testConfiguredAuthRecreatesHttpClientAfterDeserialization() throws Exception { + AuthenticationOAuth2 configuredAuth = new AuthenticationOAuth2(); + configuredAuth.configure(minimalCredentialsJson()); + + byte[] serializedAuth; + try (ByteArrayOutputStream out = new ByteArrayOutputStream(); + ObjectOutputStream objectOutputStream = new ObjectOutputStream(out)) { + objectOutputStream.writeObject(configuredAuth); + serializedAuth = out.toByteArray(); + } finally { + configuredAuth.close(); + } + + AuthenticationOAuth2 deserializedAuth; + try (ObjectInputStream objectInputStream = + new ObjectInputStream(new ByteArrayInputStream(serializedAuth))) { + deserializedAuth = (AuthenticationOAuth2) objectInputStream.readObject(); + } + try { + assertNotNull(((FlowBase) deserializedAuth.flow).getHttpClient()); + } finally { + deserializedAuth.close(); + } + } + @Test public void testConfigureWithTlsClientAuth() throws Exception { Map params = new HashMap<>(); @@ -405,4 +435,4 @@ public void testClose() throws Exception { verify(this.flow).close(); assertThrows(PulsarClientException.AlreadyClosedException.class, () -> this.auth.getAuthData()); } -} \ No newline at end of file +} From fa27e060b07b56d32187e480d3ea6e4b30cc689f Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 13 Aug 2026 07:39:27 +0300 Subject: [PATCH 171/213] [improve][misc] Upgrade Conscrypt to 2.6.2 to restore the native library glibc baseline (#26315) (cherry picked from commit 416d9d7c6435f8db67e8c66c1edefb8a0ba0ba86) --- distribution/server/src/assemble/LICENSE.bin.txt | 2 +- distribution/shell/src/assemble/LICENSE.bin.txt | 2 +- pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 2519a0dce53cc..d848d94c8f5ce 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -261,7 +261,7 @@ The Apache Software License, Version 2.0 - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.18.9.jar - com.fasterxml.jackson.module-jackson-module-parameter-names-2.18.9.jar * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.9.1.jar - * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.6.1.jar + * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.6.2.jar * Fastutil -- it.unimi.dsi-fastutil-8.5.16.jar * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.59.2.jar * Bitbucket -- org.bitbucket.b_c-jose4j-0.9.6.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 1e6cbef8c2367..3a0ad65b244d3 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -325,7 +325,7 @@ The Apache Software License, Version 2.0 - jackson-datatype-jsr310-2.18.9.jar - jackson-module-parameter-names-2.18.9.jar * Caffeine -- caffeine-2.9.1.jar - * Conscrypt -- conscrypt-openjdk-uber-2.6.1.jar + * Conscrypt -- conscrypt-openjdk-uber-2.6.2.jar * Gson - gson-2.13.2.jar * Guava diff --git a/pom.xml b/pom.xml index 02b7ae9a52c1b..b677cebb85846 100644 --- a/pom.xml +++ b/pom.xml @@ -192,7 +192,7 @@ flexible messaging model and an intuitive client API. 12.1.12 9.4.58.v20250814 - 2.6.1 + 2.6.2 2.42 1.10.62 0.16.0 From c2b7e0bf2f4c218f25fe81c875e369c451518451 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 8 Jun 2026 20:44:36 +0300 Subject: [PATCH 172/213] [improve][misc] Upgrade log4j to 2.26.0 and slf4j to 2.0.18 (#25973) (cherry picked from commit 492a231c06fb838be8e36975980145eeb3014943) --- distribution/server/src/assemble/LICENSE.bin.txt | 14 +++++++------- distribution/shell/src/assemble/LICENSE.bin.txt | 10 +++++----- pom.xml | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index d848d94c8f5ce..90be922da39e4 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -348,11 +348,11 @@ The Apache Software License, Version 2.0 - jakarta.validation-jakarta.validation-api-2.0.2.jar - javax.validation-validation-api-1.1.0.Final.jar * Log4J - - org.apache.logging.log4j-log4j-api-2.25.4.jar - - org.apache.logging.log4j-log4j-core-2.25.4.jar - - org.apache.logging.log4j-log4j-slf4j2-impl-2.25.4.jar - - org.apache.logging.log4j-log4j-web-2.25.4.jar - - org.apache.logging.log4j-log4j-layout-template-json-2.25.4.jar + - org.apache.logging.log4j-log4j-api-2.26.0.jar + - org.apache.logging.log4j-log4j-core-2.26.0.jar + - org.apache.logging.log4j-log4j-slf4j2-impl-2.26.0.jar + - org.apache.logging.log4j-log4j-web-2.26.0.jar + - org.apache.logging.log4j-log4j-layout-template-json-2.26.0.jar * Java Native Access JNA - net.java.dev.jna-jna-jpms-5.18.1.jar - net.java.dev.jna-jna-platform-jpms-5.18.1.jar @@ -577,8 +577,8 @@ BSD 2-Clause License MIT License * Java SemVer -- com.github.zafarkhaja-java-semver-0.9.0.jar -- ../licenses/LICENSE-SemVer.txt * SLF4J -- ../licenses/LICENSE-SLF4J.txt - - org.slf4j-slf4j-api-2.0.17.jar - - org.slf4j-jcl-over-slf4j-2.0.17.jar + - org.slf4j-slf4j-api-2.0.18.jar + - org.slf4j-jcl-over-slf4j-2.0.18.jar * The Checker Framework - org.checkerframework-checker-qual-3.33.0.jar * oshi diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 3a0ad65b244d3..adf7c90101e97 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -381,10 +381,10 @@ The Apache Software License, Version 2.0 - simpleclient_tracer_otel-0.16.0.jar - simpleclient_tracer_otel_agent-0.16.0.jar * Log4J - - log4j-api-2.25.4.jar - - log4j-core-2.25.4.jar - - log4j-slf4j2-impl-2.25.4.jar - - log4j-web-2.25.4.jar + - log4j-api-2.26.0.jar + - log4j-core-2.26.0.jar + - log4j-slf4j2-impl-2.26.0.jar + - log4j-web-2.26.0.jar * OpenTelemetry - opentelemetry-api-1.62.0.jar - opentelemetry-api-incubator-1.62.0-alpha.jar @@ -431,7 +431,7 @@ BSD 3-clause "New" or "Revised" License MIT License * SLF4J -- ../licenses/LICENSE-SLF4J.txt - - slf4j-api-2.0.17.jar + - slf4j-api-2.0.18.jar * The Checker Framework - checker-qual-3.33.0.jar diff --git a/pom.xml b/pom.xml index b677cebb85846..11bb1b2d283b9 100644 --- a/pom.xml +++ b/pom.xml @@ -198,9 +198,9 @@ flexible messaging model and an intuitive client API. 0.16.0 4.5.28 7.9.2 - 2.0.17 + 2.0.18 4.5.0 - 2.25.4 + 2.26.0 1.84 ${bouncycastle.version} From ae5e1c59d0e7bf260e2960c4eaea8945ba2f6562 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 14:05:20 +0300 Subject: [PATCH 173/213] [fix][sec] Bump log4j2 from 2.26.0 to 2.26.1 (#26329) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Lari Hotari (cherry picked from commit 307035b752fa74a9cb52cdf91dccc4f22e6d03b1) --- distribution/server/src/assemble/LICENSE.bin.txt | 10 +++++----- distribution/shell/src/assemble/LICENSE.bin.txt | 8 ++++---- pom.xml | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 90be922da39e4..921ab758ff19e 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -348,11 +348,11 @@ The Apache Software License, Version 2.0 - jakarta.validation-jakarta.validation-api-2.0.2.jar - javax.validation-validation-api-1.1.0.Final.jar * Log4J - - org.apache.logging.log4j-log4j-api-2.26.0.jar - - org.apache.logging.log4j-log4j-core-2.26.0.jar - - org.apache.logging.log4j-log4j-slf4j2-impl-2.26.0.jar - - org.apache.logging.log4j-log4j-web-2.26.0.jar - - org.apache.logging.log4j-log4j-layout-template-json-2.26.0.jar + - org.apache.logging.log4j-log4j-api-2.26.1.jar + - org.apache.logging.log4j-log4j-core-2.26.1.jar + - org.apache.logging.log4j-log4j-slf4j2-impl-2.26.1.jar + - org.apache.logging.log4j-log4j-web-2.26.1.jar + - org.apache.logging.log4j-log4j-layout-template-json-2.26.1.jar * Java Native Access JNA - net.java.dev.jna-jna-jpms-5.18.1.jar - net.java.dev.jna-jna-platform-jpms-5.18.1.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index adf7c90101e97..82bdf448e010b 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -381,10 +381,10 @@ The Apache Software License, Version 2.0 - simpleclient_tracer_otel-0.16.0.jar - simpleclient_tracer_otel_agent-0.16.0.jar * Log4J - - log4j-api-2.26.0.jar - - log4j-core-2.26.0.jar - - log4j-slf4j2-impl-2.26.0.jar - - log4j-web-2.26.0.jar + - log4j-api-2.26.1.jar + - log4j-core-2.26.1.jar + - log4j-slf4j2-impl-2.26.1.jar + - log4j-web-2.26.1.jar * OpenTelemetry - opentelemetry-api-1.62.0.jar - opentelemetry-api-incubator-1.62.0-alpha.jar diff --git a/pom.xml b/pom.xml index 11bb1b2d283b9..e42dac498753a 100644 --- a/pom.xml +++ b/pom.xml @@ -200,7 +200,7 @@ flexible messaging model and an intuitive client API. 7.9.2 2.0.18 4.5.0 - 2.26.0 + 2.26.1 1.84 ${bouncycastle.version} From 7cb381e3d7e13cadf9f4e451a29b8d919175167d Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Sun, 16 Aug 2026 15:14:00 +0300 Subject: [PATCH 174/213] [fix][sec] Upgrade Jackson to 2.18.10 (#26339) (cherry picked from commit f3eaa6e7d351814b9f35d23cc1a2e0bcae31fef2) --- .../server/src/assemble/LICENSE.bin.txt | 22 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 22 +++++++++---------- pom.xml | 2 +- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 921ab758ff19e..fcef3c1858a79 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -249,17 +249,17 @@ The Apache Software License, Version 2.0 - info.picocli-picocli-shell-jline3-4.7.7.jar * High Performance Primitive Collections for Java -- com.carrotsearch-hppc-0.9.1.jar * Jackson - - com.fasterxml.jackson.core-jackson-annotations-2.18.9.jar - - com.fasterxml.jackson.core-jackson-core-2.18.9.jar - - com.fasterxml.jackson.core-jackson-databind-2.18.9.jar - - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.18.9.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.18.9.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.18.9.jar - - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.18.9.jar - - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.18.9.jar - - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.18.9.jar - - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.18.9.jar - - com.fasterxml.jackson.module-jackson-module-parameter-names-2.18.9.jar + - com.fasterxml.jackson.core-jackson-annotations-2.18.10.jar + - com.fasterxml.jackson.core-jackson-core-2.18.10.jar + - com.fasterxml.jackson.core-jackson-databind-2.18.10.jar + - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.18.10.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.18.10.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.18.10.jar + - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.18.10.jar + - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.18.10.jar + - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.18.10.jar + - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.18.10.jar + - com.fasterxml.jackson.module-jackson-module-parameter-names-2.18.10.jar * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.9.1.jar * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.6.2.jar * Fastutil -- it.unimi.dsi-fastutil-8.5.16.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 82bdf448e010b..e5a6e42fd22c4 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -313,17 +313,17 @@ The Apache Software License, Version 2.0 - picocli-4.7.7.jar - picocli-shell-jline3-4.7.7.jar * Jackson - - jackson-annotations-2.18.9.jar - - jackson-core-2.18.9.jar - - jackson-databind-2.18.9.jar - - jackson-dataformat-yaml-2.18.9.jar - - jackson-jaxrs-base-2.18.9.jar - - jackson-jaxrs-json-provider-2.18.9.jar - - jackson-module-jaxb-annotations-2.18.9.jar - - jackson-module-jsonSchema-2.18.9.jar - - jackson-datatype-jdk8-2.18.9.jar - - jackson-datatype-jsr310-2.18.9.jar - - jackson-module-parameter-names-2.18.9.jar + - jackson-annotations-2.18.10.jar + - jackson-core-2.18.10.jar + - jackson-databind-2.18.10.jar + - jackson-dataformat-yaml-2.18.10.jar + - jackson-jaxrs-base-2.18.10.jar + - jackson-jaxrs-json-provider-2.18.10.jar + - jackson-module-jaxb-annotations-2.18.10.jar + - jackson-module-jsonSchema-2.18.10.jar + - jackson-datatype-jdk8-2.18.10.jar + - jackson-datatype-jsr310-2.18.10.jar + - jackson-module-parameter-names-2.18.10.jar * Caffeine -- caffeine-2.9.1.jar * Conscrypt -- conscrypt-openjdk-uber-2.6.2.jar * Gson diff --git a/pom.xml b/pom.xml index e42dac498753a..3a23235e9e878 100644 --- a/pom.xml +++ b/pom.xml @@ -209,7 +209,7 @@ flexible messaging model and an intuitive client API. 2.0.11 2.0.6 2.0.1 - 2.18.9 + 2.18.10 8.5.16 0.10.2 1.6.2 From 01edfab9a0a888e284b10b6bce5829bcb6380d84 Mon Sep 17 00:00:00 2001 From: sinan liu Date: Mon, 17 Aug 2026 19:11:13 +0800 Subject: [PATCH 175/213] [fix][broker] Prevent Key_Shared out-of-order replay starvation at the end of topic (#26268) (cherry picked from commit bcdbdaaafc427595f374965a5f7d5f00c2a5a35d) --- .../MessageRedeliveryController.java | 88 ++++++--- ...tStickyKeyDispatcherMultipleConsumers.java | 54 +++--- ...KeyDispatcherMultipleConsumersClassic.java | 6 +- .../MessageRedeliveryControllerTest.java | 172 ++++++++---------- ...ckyKeyDispatcherMultipleConsumersTest.java | 62 ++++++- 5 files changed, 228 insertions(+), 154 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryController.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryController.java index 46f1f0a535650..9e5ebcf38f432 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryController.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryController.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.service.persistent; import static org.apache.pulsar.broker.service.StickyKeyConsumerSelector.STICKY_KEY_HASH_NOT_SET; +import com.google.common.annotations.VisibleForTesting; import java.util.ArrayList; import java.util.List; import java.util.NavigableSet; @@ -43,7 +44,11 @@ public class MessageRedeliveryController { private final boolean allowOutOfOrderDelivery; private final boolean isClassicDispatcher; private final ConcurrentBitmapSortedLongPairSet messagesToRedeliver; - private final ConcurrentLongLongPairHashMap hashesToBeBlocked; + // Not final: under out-of-order delivery, whether this map is ever needed isn't known at construction time. Only a + // Key_Shared dispatcher records hashes; a plain Shared dispatcher never does. The map is therefore allocated when + // add() first receives a real hash. Classic out-of-order returns before that allocation regardless of the hash. + private ConcurrentLongLongPairHashMap positionToStickyKeyHash; + // Final by contrast: this map is needed exactly when ordering is enforced, which is known at construction time. private final ConcurrentLongLongHashMap hashesRefCount; public MessageRedeliveryController(boolean allowOutOfOrderDelivery) { @@ -55,16 +60,27 @@ public MessageRedeliveryController(boolean allowOutOfOrderDelivery, boolean isCl this.isClassicDispatcher = isClassicDispatcher; this.messagesToRedeliver = new ConcurrentBitmapSortedLongPairSet(); if (!allowOutOfOrderDelivery) { - this.hashesToBeBlocked = ConcurrentLongLongPairHashMap - .newBuilder().concurrencyLevel(2).expectedItems(128).autoShrink(true).build(); + this.positionToStickyKeyHash = newPositionToStickyKeyHashMap(); this.hashesRefCount = ConcurrentLongLongHashMap .newBuilder().concurrencyLevel(2).expectedItems(128).autoShrink(true).build(); } else { - this.hashesToBeBlocked = null; + this.positionToStickyKeyHash = null; this.hashesRefCount = null; } } + private static ConcurrentLongLongPairHashMap newPositionToStickyKeyHashMap() { + return ConcurrentLongLongPairHashMap.newBuilder() + .concurrencyLevel(2).expectedItems(128).autoShrink(true).build(); + } + + private ConcurrentLongLongPairHashMap ensurePositionToStickyKeyHashMap() { + if (positionToStickyKeyHash == null) { + positionToStickyKeyHash = newPositionToStickyKeyHashMap(); + } + return positionToStickyKeyHash; + } + public void add(long ledgerId, long entryId) { messagesToRedeliver.add(ledgerId, entryId); } @@ -74,30 +90,38 @@ public void add(long ledgerId, long entryId, long stickyKeyHash) { if (!isClassicDispatcher && stickyKeyHash == STICKY_KEY_HASH_NOT_SET) { throw new IllegalArgumentException("Sticky key hash is not set. It is required."); } - boolean inserted = hashesToBeBlocked.putIfAbsent(ledgerId, entryId, stickyKeyHash, 0); - if (!inserted) { - hashesToBeBlocked.put(ledgerId, entryId, stickyKeyHash, 0); - } else { - // Return -1 means the key was not present - long stored = hashesRefCount.get(stickyKeyHash); - hashesRefCount.put(stickyKeyHash, stored > 0 ? ++stored : 1); - } + } else if (isClassicDispatcher || stickyKeyHash == STICKY_KEY_HASH_NOT_SET) { + // Classic out-of-order dispatchers never read position hashes. Non-classic dispatchers normalize real + // sticky-key hashes away from the sentinel, so the sentinel denotes a replay position without a known hash. + messagesToRedeliver.add(ledgerId, entryId); + return; + } + ConcurrentLongLongPairHashMap positionToStickyKeyHash = ensurePositionToStickyKeyHashMap(); + boolean inserted = positionToStickyKeyHash.putIfAbsent(ledgerId, entryId, stickyKeyHash, 0); + if (!inserted) { + positionToStickyKeyHash.put(ledgerId, entryId, stickyKeyHash, 0); + } else if (!allowOutOfOrderDelivery) { + // Return -1 means the key was not present + long stored = hashesRefCount.get(stickyKeyHash); + hashesRefCount.put(stickyKeyHash, stored > 0 ? ++stored : 1); } messagesToRedeliver.add(ledgerId, entryId); } public void remove(long ledgerId, long entryId) { - if (!allowOutOfOrderDelivery) { - removeFromHashBlocker(ledgerId, entryId); - } + removeFromStickyKeyHash(ledgerId, entryId); messagesToRedeliver.remove(ledgerId, entryId); } - private void removeFromHashBlocker(long ledgerId, long entryId) { - LongPair value = hashesToBeBlocked.get(ledgerId, entryId); + private void removeFromStickyKeyHash(long ledgerId, long entryId) { + ConcurrentLongLongPairHashMap positionToStickyKeyHash = this.positionToStickyKeyHash; + if (positionToStickyKeyHash == null) { + return; + } + LongPair value = positionToStickyKeyHash.get(ledgerId, entryId); if (value != null) { - boolean removed = hashesToBeBlocked.remove(ledgerId, entryId, value.first, 0); - if (removed) { + boolean removed = positionToStickyKeyHash.remove(ledgerId, entryId, value.first, 0); + if (removed && !allowOutOfOrderDelivery) { long exists = hashesRefCount.get(value.first); if (exists == 1) { hashesRefCount.remove(value.first, exists); @@ -109,7 +133,11 @@ private void removeFromHashBlocker(long ledgerId, long entryId) { } public Long getHash(long ledgerId, long entryId) { - LongPair value = hashesToBeBlocked.get(ledgerId, entryId); + ConcurrentLongLongPairHashMap positionToStickyKeyHash = this.positionToStickyKeyHash; + if (positionToStickyKeyHash == null) { + return null; + } + LongPair value = positionToStickyKeyHash.get(ledgerId, entryId); if (value == null) { return null; } @@ -118,17 +146,18 @@ public Long getHash(long ledgerId, long entryId) { public void removeAllUpTo(long markDeleteLedgerId, long markDeleteEntryId) { boolean bitsCleared = messagesToRedeliver.removeUpTo(markDeleteLedgerId, markDeleteEntryId + 1); - // only if bits have been clear, and we are not allowing out of order delivery, we need to remove the hashes - // removing hashes is a relatively expensive operation, so we should only do it when necessary - if (bitsCleared && !allowOutOfOrderDelivery) { + // Only remove the hashes when bits have been cleared. Removing hashes is a relatively expensive operation, + // so we should only do it when necessary. + ConcurrentLongLongPairHashMap positionToStickyKeyHash = this.positionToStickyKeyHash; + if (bitsCleared && positionToStickyKeyHash != null && !positionToStickyKeyHash.isEmpty()) { List keysToRemove = new ArrayList<>(); - hashesToBeBlocked.forEach((ledgerId, entryId, stickyKeyHash, none) -> { + positionToStickyKeyHash.forEach((ledgerId, entryId, stickyKeyHash, none) -> { if (ledgerId < markDeleteLedgerId || (ledgerId == markDeleteLedgerId && entryId <= markDeleteEntryId)) { keysToRemove.add(new LongPair(ledgerId, entryId)); } }); for (LongPair longPair : keysToRemove) { - removeFromHashBlocker(longPair.first, longPair.second); + removeFromStickyKeyHash(longPair.first, longPair.second); } } } @@ -138,8 +167,10 @@ public boolean isEmpty() { } public void clear() { + if (positionToStickyKeyHash != null) { + positionToStickyKeyHash.clear(); + } if (!allowOutOfOrderDelivery) { - hashesToBeBlocked.clear(); hashesRefCount.clear(); } messagesToRedeliver.clear(); @@ -197,4 +228,9 @@ public NavigableSet getMessagesToReplayNow(int maxMessagesToRead, Pred public int size() { return messagesToRedeliver.size(); } + + @VisibleForTesting + boolean isPositionToStickyKeyHashInitialized() { + return positionToStickyKeyHash != null; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java index 436e7c43e010f..6d3ee0da11137 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java @@ -316,28 +316,23 @@ protected synchronized boolean trySendMessagesToConsumers(ReadType readType, Lis acquirePermitsForDeliveredMessages(topic, cursor, totalEntries, totalMessagesSent, totalBytesSent); // trigger read more messages if necessary - if (triggerLookAhead.booleanValue() && (allowOutOfOrderDelivery || cursor.hasMoreEntries())) { + if (triggerLookAhead.booleanValue() && cursor.hasMoreEntries()) { // When all messages get filtered and no messages are sent, we should read more entries, "look ahead" // so that a possible next batch of messages might contain messages that can be dispatched. // This is done only when there's a consumer with available permits, and it's not able to make progress // because of blocked hashes. Without this rule we would be looking ahead in the stream while the // new consumers are not ready to accept the new messages, // therefore would be most likely only increase the distance between read-position and mark-delete position. - // When ordered delivery is required, look-ahead is engaged only when the cursor has more entries. - // Otherwise the next readMoreEntries call would skip replaying the replay queue and pulling due messages - // from the delayed delivery tracker, and instead issue a normal read that waits at the end of the topic - // for new entries. That would leave deliverable messages stuck in the replay queue or the delayed - // delivery tracker until an unrelated event (such as a consumer flow request) triggers another read, - // stalling dispatch (issue #21554). - // When out-of-order delivery is allowed, look-ahead is engaged unconditionally, as before. In that mode - // the replay queue doesn't track sticky key hashes, so the replay position filter cannot exclude - // messages for consumers without available permits, and each replay would re-read and discard the same - // undispatchable messages. Ending the cycle with a look-ahead attempt (that waits at the end of the - // topic when there is nothing to read) prevents such repeated read-and-discard loops. + // Look-ahead is engaged only when the cursor has more entries. Otherwise the next readMoreEntries call + // would skip replaying the replay queue and pulling due messages from the delayed delivery tracker, and + // instead issue a normal read that waits at the end of the topic for new entries. That would leave + // deliverable messages stuck in the replay queue or the delayed delivery tracker until an unrelated event + // (such as a consumer flow request) triggers another read, stalling dispatch (issue #21554). + // The former "allowOutOfOrderDelivery ||" escape here was dropped once ReplayPositionFilter started + // filtering out-of-order replay positions by available permits; don't re-add it without removing that. skipNextReplayToTriggerLookAhead = true; // skip backoff delay before reading ahead in the "look ahead" mode to prevent any additional latency - // only skip the delay if there are more entries to read - skipNextBackoff = cursor.hasMoreEntries(); + skipNextBackoff = true; return true; } @@ -566,19 +561,20 @@ private class ReplayPositionFilter implements Predicate { private final Map availablePermitsMap = new HashMap<>(); // tracks the hashes that have been blocked during the filtering // it is necessary to block all later messages after a hash gets blocked so that ordering is preserved - private final Set alreadyBlockedHashes = new HashSet<>(); + private final Set hashesBlockedForOrdering = new HashSet<>(); @Override public boolean test(Position position) { - // if out of order delivery is allowed, then any position will be replayed - if (isAllowOutOfOrderDelivery()) { - return true; - } - // lookup the sticky key hash for the entry at the replay position + // Out-of-order delivery relaxes ordering, not routing: filterAndGroupEntriesForDispatching selects the + // owning consumer by hash in both modes, so the hash is needed in both. It feeds the permit check below, + // which keeps this read's bounded budget (see MessageRedeliveryController#getMessagesToReplayNow) off + // positions that dispatch would only push straight back into the replay queue. Under out-of-order delivery, + // that check and the no-consumer check are the only live rejections here, and they made the + // "allowOutOfOrderDelivery ||" look-ahead escape removable. Long stickyKeyHash = redeliveryMessages.getHash(position.getLedgerId(), position.getEntryId()); if (stickyKeyHash == null) { - // the sticky key hash is missing for delayed messages, the filtering will happen at the time of - // dispatch after reading the entry from the ledger + // The sticky key hash is missing for delayed messages and positions added through hash-less + // redelivery paths. Filtering will happen at dispatch time after reading the entry from the ledger. if (log.isDebugEnabled()) { log.debug("[{}] replay of entry at position {} doesn't contain sticky key hash.", name, position); } @@ -586,7 +582,7 @@ public boolean test(Position position) { } // check if the hash is already blocked, if so, then replaying of the position should be skipped // to preserve ordering - if (alreadyBlockedHashes.contains(stickyKeyHash)) { + if (hashesBlockedForOrdering.contains(stickyKeyHash)) { return false; } @@ -594,7 +590,7 @@ public boolean test(Position position) { Consumer consumer = selector.select(stickyKeyHash.intValue()); // skip replaying the message position if there's no assigned consumer if (consumer == null) { - alreadyBlockedHashes.add(stickyKeyHash); + blockStickyKeyHashIfOrderingRequired(stickyKeyHash); return false; } @@ -604,20 +600,26 @@ public boolean test(Position position) { k -> new MutableInt(getAvailablePermits(consumer))); // skip replaying the message position if the consumer has no available permits if (availablePermits.intValue() <= 0) { - alreadyBlockedHashes.add(stickyKeyHash); + blockStickyKeyHashIfOrderingRequired(stickyKeyHash); return false; } if (drainingHashesRequired && drainingHashesTracker.shouldBlockStickyKeyHash(consumer, stickyKeyHash.intValue())) { // the hash is draining and the consumer is not the draining consumer - alreadyBlockedHashes.add(stickyKeyHash); + blockStickyKeyHashIfOrderingRequired(stickyKeyHash); return false; } availablePermits.decrement(); return true; } + + private void blockStickyKeyHashIfOrderingRequired(long stickyKeyHash) { + if (!allowOutOfOrderDelivery) { + hashesBlockedForOrdering.add(stickyKeyHash); + } + } } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersClassic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersClassic.java index 500f6a9472c57..16e1f3d64370c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersClassic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersClassic.java @@ -546,8 +546,7 @@ private int getAvailablePermits(Consumer c) { @Override protected synchronized NavigableSet filterOutEntriesWillBeDiscarded(NavigableSet src) { - // The variable "hashesToBeBlocked" and "recentlyJoinedConsumers" will be null if "isAllowOutOfOrderDelivery()", - // So skip this filter out. + // Keep the classic out-of-order behavior: replay positions are not filtered before reading. if (isAllowOutOfOrderDelivery()) { return src; } @@ -595,8 +594,7 @@ protected synchronized NavigableSet filterOutEntriesWillBeDiscarded(Na */ @Override protected boolean hasConsumersNeededNormalRead() { - // The variable "hashesToBeBlocked" and "recentlyJoinedConsumers" will be null if "isAllowOutOfOrderDelivery()", - // So the method "filterOutEntriesWillBeDiscarded" will filter out nothing, just return "true" here. + // Classic out-of-order replay filtering is bypassed, so normal reads do not need the ordered-mode escape check. if (isAllowOutOfOrderDelivery()) { return true; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryControllerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryControllerTest.java index bc73ba64d5006..03c7e6cf439e3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryControllerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageRedeliveryControllerTest.java @@ -18,20 +18,16 @@ */ package org.apache.pulsar.broker.service.persistent; +import static org.apache.pulsar.broker.service.StickyKeyConsumerSelector.STICKY_KEY_HASH_NOT_SET; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertEqualsNoOrder; import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; -import java.lang.reflect.Field; import java.util.Set; import java.util.TreeSet; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.PositionFactory; -import org.apache.bookkeeper.util.collections.ConcurrentLongLongHashMap; -import org.apache.pulsar.common.util.collections.ConcurrentLongLongPairHashMap; -import org.apache.pulsar.utils.ConcurrentBitmapSortedLongPairSet; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -43,103 +39,61 @@ public Object[][] dataProvider() { } @Test(dataProvider = "allowOutOfOrderDelivery", timeOut = 10000) - public void testAddAndRemove(boolean allowOutOfOrderDelivery) throws Exception { + public void testAddAndRemove(boolean allowOutOfOrderDelivery) { MessageRedeliveryController controller = new MessageRedeliveryController(allowOutOfOrderDelivery); - Field messagesToRedeliverField = MessageRedeliveryController.class.getDeclaredField("messagesToRedeliver"); - messagesToRedeliverField.setAccessible(true); - ConcurrentBitmapSortedLongPairSet messagesToRedeliver = - (ConcurrentBitmapSortedLongPairSet) messagesToRedeliverField.get(controller); - - Field hashesToBeBlockedField = MessageRedeliveryController.class.getDeclaredField("hashesToBeBlocked"); - hashesToBeBlockedField.setAccessible(true); - ConcurrentLongLongPairHashMap hashesToBeBlocked = (ConcurrentLongLongPairHashMap) hashesToBeBlockedField - .get(controller); - - Field hashesRefCountField = MessageRedeliveryController.class.getDeclaredField("hashesRefCount"); - hashesRefCountField.setAccessible(true); - ConcurrentLongLongHashMap hashesRefCount = (ConcurrentLongLongHashMap) hashesRefCountField.get(controller); - - if (allowOutOfOrderDelivery) { - assertNull(hashesToBeBlocked); - assertNull(hashesRefCount); - } else { - assertNotNull(hashesToBeBlocked); - assertNotNull(hashesRefCount); - } - + assertEquals(controller.isPositionToStickyKeyHashInitialized(), !allowOutOfOrderDelivery); assertTrue(controller.isEmpty()); - assertEquals(messagesToRedeliver.size(), 0); - if (!allowOutOfOrderDelivery) { - assertEquals(hashesToBeBlocked.size(), 0); - assertEquals(hashesRefCount.size(), 0); - } + assertEquals(controller.size(), 0); controller.add(1, 1); controller.add(1, 2); assertFalse(controller.isEmpty()); - assertEquals(messagesToRedeliver.size(), 2); - assertTrue(messagesToRedeliver.contains(1, 1)); - assertTrue(messagesToRedeliver.contains(1, 2)); - if (!allowOutOfOrderDelivery) { - assertEquals(hashesToBeBlocked.size(), 0); - assertFalse(hashesToBeBlocked.containsKey(1, 1)); - assertFalse(hashesToBeBlocked.containsKey(1, 2)); - assertEquals(hashesRefCount.size(), 0); - } + assertEquals(controller.size(), 2); + assertNull(controller.getHash(1, 1)); + assertNull(controller.getHash(1, 2)); + assertEquals(controller.isPositionToStickyKeyHashInitialized(), !allowOutOfOrderDelivery); controller.remove(1, 1); controller.remove(1, 2); assertTrue(controller.isEmpty()); - assertEquals(messagesToRedeliver.size(), 0); - assertFalse(messagesToRedeliver.contains(1, 1)); - assertFalse(messagesToRedeliver.contains(1, 2)); - if (!allowOutOfOrderDelivery) { - assertEquals(hashesToBeBlocked.size(), 0); - assertEquals(hashesRefCount.size(), 0); - } + assertEquals(controller.size(), 0); controller.add(2, 1, 100); controller.add(2, 2, 101); controller.add(2, 3, 101); assertFalse(controller.isEmpty()); - assertEquals(messagesToRedeliver.size(), 3); - assertTrue(messagesToRedeliver.contains(2, 1)); - assertTrue(messagesToRedeliver.contains(2, 2)); - assertTrue(messagesToRedeliver.contains(2, 3)); + assertEquals(controller.size(), 3); + assertTrue(controller.isPositionToStickyKeyHashInitialized()); + assertEquals(controller.getHash(2, 1), Long.valueOf(100)); + assertEquals(controller.getHash(2, 2), Long.valueOf(101)); + assertEquals(controller.getHash(2, 3), Long.valueOf(101)); if (!allowOutOfOrderDelivery) { - assertEquals(hashesToBeBlocked.size(), 3); - assertEquals(hashesToBeBlocked.get(2, 1).first, 100); - assertEquals(hashesToBeBlocked.get(2, 2).first, 101); - assertEquals(hashesToBeBlocked.get(2, 3).first, 101); - assertEquals(hashesRefCount.size(), 2); - assertEquals(hashesRefCount.get(100), 1); - assertEquals(hashesRefCount.get(101), 2); + assertTrue(controller.containsStickyKeyHash(100)); + assertTrue(controller.containsStickyKeyHash(101)); } controller.remove(2, 1); controller.remove(2, 2); + assertEquals(controller.size(), 1); + assertNull(controller.getHash(2, 1)); + assertNull(controller.getHash(2, 2)); + assertEquals(controller.getHash(2, 3), Long.valueOf(101)); if (!allowOutOfOrderDelivery) { - assertEquals(hashesToBeBlocked.size(), 1); - assertEquals(hashesToBeBlocked.get(2, 3).first, 101); - assertEquals(hashesRefCount.size(), 1); - assertEquals(hashesRefCount.get(100), -1); - assertEquals(hashesRefCount.get(101), 1); + assertFalse(controller.containsStickyKeyHash(100)); + assertTrue(controller.containsStickyKeyHash(101)); } controller.clear(); assertTrue(controller.isEmpty()); - assertEquals(messagesToRedeliver.size(), 0); - assertTrue(messagesToRedeliver.isEmpty()); + assertEquals(controller.size(), 0); + assertNull(controller.getHash(2, 3)); if (!allowOutOfOrderDelivery) { - assertEquals(hashesToBeBlocked.size(), 0); - assertTrue(hashesToBeBlocked.isEmpty()); - assertEquals(hashesRefCount.size(), 0); - assertTrue(hashesRefCount.isEmpty()); + assertFalse(controller.containsStickyKeyHash(101)); } controller.add(2, 2, 201); @@ -151,43 +105,71 @@ public void testAddAndRemove(boolean allowOutOfOrderDelivery) throws Exception { controller.add(1, 1, 100); controller.removeAllUpTo(1, 3); - assertEquals(messagesToRedeliver.size(), 4); - assertTrue(messagesToRedeliver.contains(2, 1)); - assertTrue(messagesToRedeliver.contains(2, 2)); - assertTrue(messagesToRedeliver.contains(3, 1)); - assertTrue(messagesToRedeliver.contains(3, 2)); + assertEquals(controller.size(), 4); + assertNull(controller.getHash(1, 1)); + assertNull(controller.getHash(1, 2)); + assertNull(controller.getHash(1, 3)); + assertEquals(controller.getHash(2, 1), Long.valueOf(200)); + assertEquals(controller.getHash(2, 2), Long.valueOf(201)); + assertEquals(controller.getHash(3, 1), Long.valueOf(300)); + assertEquals(controller.getHash(3, 2), Long.valueOf(301)); if (!allowOutOfOrderDelivery) { - assertEquals(hashesToBeBlocked.size(), 4); - assertEquals(hashesToBeBlocked.get(2, 1).first, 200); - assertEquals(hashesToBeBlocked.get(2, 2).first, 201); - assertEquals(hashesToBeBlocked.get(3, 1).first, 300); - assertEquals(hashesToBeBlocked.get(3, 2).first, 301); - assertEquals(hashesRefCount.size(), 4); - assertEquals(hashesRefCount.get(200), 1); - assertEquals(hashesRefCount.get(201), 1); - assertEquals(hashesRefCount.get(300), 1); - assertEquals(hashesRefCount.get(301), 1); + assertFalse(controller.containsStickyKeyHash(100)); + assertFalse(controller.containsStickyKeyHash(101)); + assertTrue(controller.containsStickyKeyHash(200)); + assertTrue(controller.containsStickyKeyHash(201)); + assertTrue(controller.containsStickyKeyHash(300)); + assertTrue(controller.containsStickyKeyHash(301)); } controller.removeAllUpTo(3, 1); - assertEquals(messagesToRedeliver.size(), 1); - assertTrue(messagesToRedeliver.contains(3, 2)); + assertEquals(controller.size(), 1); + assertNull(controller.getHash(2, 1)); + assertNull(controller.getHash(2, 2)); + assertNull(controller.getHash(3, 1)); + assertEquals(controller.getHash(3, 2), Long.valueOf(301)); if (!allowOutOfOrderDelivery) { - assertEquals(hashesToBeBlocked.size(), 1); - assertEquals(hashesToBeBlocked.get(3, 2).first, 301); - assertEquals(hashesRefCount.size(), 1); - assertEquals(hashesRefCount.get(301), 1); + assertFalse(controller.containsStickyKeyHash(200)); + assertFalse(controller.containsStickyKeyHash(201)); + assertFalse(controller.containsStickyKeyHash(300)); + assertTrue(controller.containsStickyKeyHash(301)); } controller.removeAllUpTo(5, 10); assertTrue(controller.isEmpty()); - assertEquals(messagesToRedeliver.size(), 0); + assertEquals(controller.size(), 0); + assertNull(controller.getHash(3, 2)); if (!allowOutOfOrderDelivery) { - assertEquals(hashesToBeBlocked.size(), 0); - assertEquals(hashesRefCount.size(), 0); + assertFalse(controller.containsStickyKeyHash(301)); } } + @Test(timeOut = 10000) + public void testOutOfOrderSentinelHashDoesNotInitializePositionHashMap() { + MessageRedeliveryController controller = new MessageRedeliveryController(true); + + controller.add(1, 1, STICKY_KEY_HASH_NOT_SET); + + assertFalse(controller.isPositionToStickyKeyHashInitialized()); + assertEquals(controller.size(), 1); + assertNull(controller.getHash(1, 1)); + + controller.removeAllUpTo(1, 1); + assertTrue(controller.isEmpty()); + assertFalse(controller.isPositionToStickyKeyHashInitialized()); + } + + @Test(timeOut = 10000) + public void testClassicOutOfOrderDoesNotInitializePositionHashMap() { + MessageRedeliveryController controller = new MessageRedeliveryController(true, true); + + controller.add(1, 1, 100); + + assertFalse(controller.isPositionToStickyKeyHashInitialized()); + assertEquals(controller.size(), 1); + assertNull(controller.getHash(1, 1)); + } + @Test(dataProvider = "allowOutOfOrderDelivery", timeOut = 10000) public void testContainsStickyKeyHashes(boolean allowOutOfOrderDelivery) throws Exception { MessageRedeliveryController controller = new MessageRedeliveryController(allowOutOfOrderDelivery); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java index 50007b5c86197..3a5c73315a011 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java @@ -463,6 +463,11 @@ public void testSkipRedeliverTemporally() throws InterruptedException { allEntries.forEach(Entry::release); } + @DataProvider(name = "allowOutOfOrderDelivery") + private Object[][] allowOutOfOrderDelivery() { + return new Object[][] { { false }, { true } }; + } + /** * Reproduces the dispatch stall behind the flaky * KeySharedSubscriptionTest.testContinueDispatchMessagesWhenMessageDelayed (issue #21554). @@ -473,14 +478,15 @@ public void testSkipRedeliverTemporally() throws InterruptedException { * in the replay queue for consumers with available permits were then stuck until an unrelated event, such as a * consumer flow request, triggered another read. */ - @Test(timeOut = 30000) - public void testLookAheadNotEngagedWhenCursorHasNoMoreEntries() throws Exception { + @Test(dataProvider = "allowOutOfOrderDelivery", timeOut = 30000) + public void testLookAheadNotEngagedWhenCursorHasNoMoreEntries(boolean allowOutOfOrderDelivery) throws Exception { persistentDispatcher.close(); // the mocked executor doesn't support schedule(), so run the rescheduled read directly persistentDispatcher = new PersistentStickyKeyDispatcherMultipleConsumers( topicMock, cursorMock, subscriptionMock, configMock, - new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT)) { + new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT) + .setAllowOutOfOrderDelivery(allowOutOfOrderDelivery)) { @Override protected void reScheduleReadInMs(long readAfterMs) { orderedExecutor.execute(this::readMoreEntries); @@ -562,6 +568,56 @@ protected void reScheduleReadInMs(long readAfterMs) { allEntries.forEach(Entry::release); } + @Test(timeOut = 10000) + public void testOutOfOrderReplayFilterDoesNotSpendReadBudgetOnConsumerWithoutPermits() { + persistentDispatcher.close(); + persistentDispatcher = new PersistentStickyKeyDispatcherMultipleConsumers( + topicMock, cursorMock, subscriptionMock, configMock, + new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT) + .setAllowOutOfOrderDelivery(true)); + + persistentDispatcher.addConsumer(consumerMock).join(); + + Consumer slowConsumer = createMockConsumer(); + doReturn("consumer2").when(slowConsumer).consumerName(); + doReturn(0).when(slowConsumer).getAvailablePermits(); + persistentDispatcher.addConsumer(slowConsumer).join(); + + StickyKeyConsumerSelector selector = persistentDispatcher.getSelector(); + String keyForConsumer = generateKeyForConsumer(selector, consumerMock); + String keyForSlowConsumer = generateKeyForConsumer(selector, slowConsumer); + Entry entry1 = createEntry(1, 1, "message1", 1, keyForSlowConsumer); + Entry entry2 = createEntry(1, 2, "message2", 2, keyForSlowConsumer); + Entry entry3 = createEntry(1, 3, "message3", 3, keyForConsumer); + List entries = List.of(entry1, entry2, entry3); + + try { + for (Entry entry : entries) { + ((EntryImpl) entry).retain(); + persistentDispatcher.addEntryToReplay(entry); + } + + Set positions = persistentDispatcher.getMessagesToReplayNow(1, Long.MAX_VALUE); + assertThat(positions).containsExactly(entry3.getPosition()); + } finally { + entries.forEach(Entry::release); + } + } + + @Test(timeOut = 10000) + public void testOutOfOrderReplayFilterIncludesPositionWithoutStickyKeyHash() { + persistentDispatcher.close(); + persistentDispatcher = new PersistentStickyKeyDispatcherMultipleConsumers( + topicMock, cursorMock, subscriptionMock, configMock, + new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT) + .setAllowOutOfOrderDelivery(true)); + + assertTrue(persistentDispatcher.addMessageToReplay(1, 1)); + + Set positions = persistentDispatcher.getMessagesToReplayNow(1, Long.MAX_VALUE); + assertThat(positions).containsExactly(PositionFactory.create(1, 1)); + } + @Test(timeOut = 30000) public void testMessageRedelivery() throws Exception { final List actualEntriesToConsumer1 = new CopyOnWriteArrayList<>(); From 284dfc60781d0c673caf93e094598d1bfaeef21b Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 27 Jul 2026 11:30:41 +0300 Subject: [PATCH 176/213] [fix][test] Fix flaky AdminApiTest caused by leaked brokerShutdownTimeoutMs (#26249) (cherry picked from commit 6a109ac2e25c1b9d9d54d9121945ea00f57c79e2) Backport note: branch-4.0 still carries V1AdminApiTest, the copy of AdminApiTest that PIP-457 (#25304) removed from master, and it cannot inherit this fix. It has the same defect: testGetDynamicLocalConfiguration leaves the brokerShutdownTimeoutMs dynamic configuration at 10ms, which poisons the stopBroker() in testInvalidDynamicConfigContentInZK and turns the @AfterMethod (reset) into a class-wide skip. The same fix is therefore applied to V1AdminApiTest as well. In V1AdminApiTest the added wait is placed before the pre-existing in-memory setBrokerShutdownTimeoutMs(defaultValue) statement instead of at the end of the method: that statement restores only the in-memory value and never removes the dynamic configuration, so appending the wait after it would make the wait unsatisfiable. --- .../pulsar/broker/admin/AdminApiTest.java | 31 +++++++++++++++++ .../broker/admin/v1/V1AdminApiTest.java | 33 +++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java index 0fdc42957c130..9212db6634053 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java @@ -159,6 +159,8 @@ public class AdminApiTest extends MockedPulsarServiceBaseTest { private static final Logger LOG = LoggerFactory.getLogger(AdminApiTest.class); + private static final String BROKER_SHUTDOWN_TIMEOUT_MS = "brokerShutdownTimeoutMs"; + private MockedPulsarService mockPulsarSetup; private PulsarService otherPulsar; @@ -221,6 +223,14 @@ private void setupConfigAndStart(java.util.function.Consumer assertNotEquals(pulsar.getConfiguration().getBrokerShutdownTimeoutMs(), overriddenTimeoutMs)); + } + private void setupClusters() throws PulsarAdminException { admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test")); @@ -759,6 +788,8 @@ public void testGetDynamicLocalConfiguration() throws Exception { admin.brokers().updateDynamicConfiguration(configName, Long.toString(shutdownTime)); // Now, znode is created: updateConfigurationAndRegisterListeners and check if configuration updated assertEquals(Long.parseLong(admin.brokers().getAllDynamicConfigurations().get(configName)), shutdownTime); + // wait until the broker has applied the value, so that the @AfterMethod always has to restore it + Awaitility.await().until(() -> pulsar.getConfiguration().getBrokerShutdownTimeoutMs() == shutdownTime); } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1AdminApiTest.java index 2d1a0acab843f..48e4dcbbb0174 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1AdminApiTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1AdminApiTest.java @@ -113,6 +113,7 @@ import org.apache.pulsar.compaction.Compactor; import org.apache.pulsar.compaction.PulsarCompactionServiceFactory; import org.apache.pulsar.metadata.cache.impl.MetadataCacheImpl; +import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; @@ -128,6 +129,8 @@ public class V1AdminApiTest extends MockedPulsarServiceBaseTest { private static final Logger LOG = LoggerFactory.getLogger(V1AdminApiTest.class); + private static final String BROKER_SHUTDOWN_TIMEOUT_MS = "brokerShutdownTimeoutMs"; + private MockedPulsarService mockPulsarSetup; private PulsarService otherPulsar; @@ -189,6 +192,14 @@ protected void customizeMainPulsarTestContextBuilder(PulsarTestContext.Builder p @AfterMethod(alwaysRun = true) public void reset() throws Exception { + if (pulsar == null || !pulsar.isRunning()) { + // A test method left the broker stopped, for example because stopBroker() failed. Throwing from + // here is a TestNG configuration failure, which makes TestNG skip every remaining test method of + // this class. TestRetrySupport recreates the shared test context before the next test method. + log.warn("Broker isn't running, skipping the cleanup of the previous test method"); + return; + } + restoreBrokerShutdownTimeout(); pulsar.getConfiguration().setForceDeleteNamespaceAllowed(true); for (String tenant : admin.tenants().getTenants()) { for (String namespace : admin.namespaces().getNamespaces(tenant)) { @@ -211,6 +222,25 @@ public void reset() throws Exception { admin.namespaces().createNamespace("prop-xyz/use/ns1"); } + /** + * Several test methods of this class lower the {@code brokerShutdownTimeoutMs} dynamic configuration to 10ms. + * The broker applies dynamic configuration changes asynchronously, so when the value is left behind it can + * become effective while a later test method is shutting the broker down in stopBroker() or restartBroker(). + * PulsarService#close then fails with "Timeout in close" and leaves the test holding a broker that is no + * longer listening, which fails this @AfterMethod and skips the remaining test methods of the class. + */ + private void restoreBrokerShutdownTimeout() throws Exception { + String overriddenValue = admin.brokers().getAllDynamicConfigurations().get(BROKER_SHUTDOWN_TIMEOUT_MS); + if (overriddenValue == null) { + return; + } + long overriddenTimeoutMs = Long.parseLong(overriddenValue); + // removing the dynamic configuration makes the broker restore the value it was started with + admin.brokers().deleteDynamicConfiguration(BROKER_SHUTDOWN_TIMEOUT_MS); + Awaitility.await().untilAsserted( + () -> assertNotEquals(pulsar.getConfiguration().getBrokerShutdownTimeoutMs(), overriddenTimeoutMs)); + } + @DataProvider(name = "numBundles") public static Object[][] numBundles() { return new Object[][] { { 1 }, { 4 } }; @@ -622,6 +652,9 @@ public void testGetDynamicLocalConfiguration() throws Exception { admin.brokers().updateDynamicConfiguration(configName, Long.toString(shutdownTime)); // Now, znode is created: updateConfigurationAndregisterListeners and check if configuration updated assertEquals(Long.parseLong(admin.brokers().getAllDynamicConfigurations().get(configName)), shutdownTime); + // wait until the broker has applied the value before restoring it below, so that the asynchronous + // update cannot set it back to shutdownTime once this test method has completed + Awaitility.await().until(() -> pulsar.getConfiguration().getBrokerShutdownTimeoutMs() == shutdownTime); pulsar.getConfiguration().setBrokerShutdownTimeoutMs(defaultValue); } From c7c4a74ce8a5e192f1ac0c382e5f0a61e9b5844a Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Wed, 19 Aug 2026 04:19:49 +0800 Subject: [PATCH 177/213] [fix][offload] Normalize offloader cache directory path (#26372) (cherry picked from commit ae07c33d6cbfe37cd9111e02485797cacd9aef35) --- .../mledger/offload/OffloadersCache.java | 4 ++- .../mledger/offload/OffloadersCacheTest.java | 32 ++++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloadersCache.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloadersCache.java index 9f2f5860d4ebd..03c87713566c4 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloadersCache.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloadersCache.java @@ -19,6 +19,7 @@ package org.apache.bookkeeper.mledger.offload; import java.io.IOException; +import java.nio.file.Paths; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import lombok.extern.slf4j.Slf4j; @@ -41,7 +42,8 @@ public class OffloadersCache implements AutoCloseable { * @throws IOException when fail to retrieve the pulsar offloader class */ public Offloaders getOrLoadOffloaders(String offloadersPath, String narExtractionDirectory) { - return loadedOffloaders.computeIfAbsent(offloadersPath, + String normalizedOffloadersPath = Paths.get(offloadersPath).toAbsolutePath().normalize().toString(); + return loadedOffloaders.computeIfAbsent(normalizedOffloadersPath, (directory) -> { try { return OffloaderUtils.searchForOffloaders(directory, narExtractionDirectory); diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/offload/OffloadersCacheTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/offload/OffloadersCacheTest.java index f20b9b9cbbf2e..9879019471565 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/offload/OffloadersCacheTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/offload/OffloadersCacheTest.java @@ -19,7 +19,10 @@ package org.apache.bookkeeper.mledger.offload; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.testng.Assert.assertSame; +import java.nio.file.Paths; import org.mockito.MockedStatic; import org.mockito.Mockito; import org.testng.annotations.Test; @@ -29,9 +32,10 @@ public class OffloadersCacheTest { @Test public void testLoadsOnlyOnce() throws Exception { Offloaders expectedOffloaders = new Offloaders(); + String normalizedPath = Paths.get("./offloaders").toAbsolutePath().normalize().toString(); try (MockedStatic offloaderUtils = Mockito.mockStatic(OffloaderUtils.class)) { - offloaderUtils.when(() -> OffloaderUtils.searchForOffloaders(eq("./offloaders"), eq("/tmp"))) + offloaderUtils.when(() -> OffloaderUtils.searchForOffloaders(eq(normalizedPath), eq("/tmp"))) .thenReturn(expectedOffloaders); OffloadersCache cache = new OffloadersCache(); @@ -47,4 +51,30 @@ public void testLoadsOnlyOnce() throws Exception { assertSame(offloaders2, expectedOffloaders, "The offloaders should be the mocked one."); } } + + @Test + public void testEquivalentPathsLoadOnlyOnce() throws Exception { + String relativePath = "./offloaders"; + String normalizedPath = Paths.get(relativePath).toAbsolutePath().normalize().toString(); + Offloaders expectedOffloaders = new Offloaders(); + + try (MockedStatic offloaderUtils = Mockito.mockStatic(OffloaderUtils.class)) { + offloaderUtils.when(() -> OffloaderUtils.searchForOffloaders(eq(relativePath), eq("/tmp"))) + .thenReturn(expectedOffloaders); + offloaderUtils.when(() -> OffloaderUtils.searchForOffloaders(eq(normalizedPath), eq("/tmp"))) + .thenReturn(expectedOffloaders); + + OffloadersCache cache = new OffloadersCache(); + + Offloaders offloaders1 = cache.getOrLoadOffloaders(relativePath, "/tmp"); + Offloaders offloaders2 = cache.getOrLoadOffloaders(normalizedPath, "/tmp"); + + assertSame(offloaders1, expectedOffloaders, "The relative path should load the mocked offloaders."); + assertSame(offloaders2, expectedOffloaders, "The absolute path should reuse the cached offloaders."); + offloaderUtils.verify( + () -> OffloaderUtils.searchForOffloaders(eq(normalizedPath), eq("/tmp")), times(1)); + offloaderUtils.verify( + () -> OffloaderUtils.searchForOffloaders(eq(relativePath), eq("/tmp")), never()); + } + } } From 68ddfdd4595ebacfe980c5b529db9c5ff0746930 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 19 Aug 2026 01:28:52 +0300 Subject: [PATCH 178/213] [fix][client] Apply no-memory-limit producer queue defaults at producer creation (#26342) (cherry picked from commit b660d70566d0e049b3a459a272874b8a47a41c09) --- .../client/api/ProducerQueueSizeTest.java | 294 ++++++++++++++++++ .../pulsar/client/api/ProducerBuilder.java | 15 +- .../client/impl/ProducerBuilderImpl.java | 27 +- .../pulsar/client/impl/PulsarClientImpl.java | 81 ++++- .../impl/conf/ProducerConfigurationData.java | 12 +- .../client/impl/ProducerBuilderImplTest.java | 42 ++- .../functions/instance/ContextImplTest.java | 5 + 7 files changed, 460 insertions(+), 16 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerQueueSizeTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerQueueSizeTest.java index 0c470ae587512..33badf3e52fe6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerQueueSizeTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerQueueSizeTest.java @@ -18,10 +18,14 @@ */ package org.apache.pulsar.client.api; +import static org.assertj.core.api.Assertions.assertThat; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import lombok.Cleanup; +import org.apache.pulsar.client.impl.ProducerBase; +import org.apache.pulsar.client.impl.conf.ProducerConfigurationData; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; @@ -29,6 +33,14 @@ public class ProducerQueueSizeTest extends ProducerConsumerBase { + /** + * The bounds {@code PulsarClientImpl} falls back to when the client memory limit is disabled. + * Duplicated here on purpose: these are a documented client default, so a change to them should + * break a test rather than pass silently. + */ + private static final int NO_MEMORY_LIMIT_MAX_PENDING_MESSAGES = 1000; + private static final int NO_MEMORY_LIMIT_MAX_PENDING_MESSAGES_ACROSS_PARTITIONS = 50000; + @BeforeMethod @Override protected void setup() throws Exception { @@ -42,6 +54,15 @@ protected void cleanup() throws Exception { super.internalCleanup(); } + private static ProducerConfigurationData confOf(Producer producer) { + return ((ProducerBase) producer).getConfiguration(); + } + + @DataProvider(name = "partitioned") + public Object[][] partitioned() { + return new Object[][]{{Boolean.FALSE}, {Boolean.TRUE}}; + } + @DataProvider(name = "matrix") public Object[][] matrix() { return new Object[][]{ @@ -85,4 +106,277 @@ public void testRemoveMaxQueueLimit(boolean blockIfQueueFull, boolean partitione f.get(); } } + + /** + * A client with the memory limit disabled has no byte-based backpressure, so producers must fall + * back to a bounded pending-message queue. This has to hold for the no-argument + * {@code newProducer()} overload as well, not just {@code newProducer(Schema)}. + */ + @Test + public void testNoArgNewProducerIsBoundedWhenMemoryLimitDisabled() throws Exception { + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(brokerUrl.toString()) + .memoryLimit(0, SizeUnit.BYTES) + .build(); + + @Cleanup + Producer producer = client.newProducer().topic(newTopicName()).create(); + + assertThat(confOf(producer).getMaxPendingMessages()) + .isEqualTo(NO_MEMORY_LIMIT_MAX_PENDING_MESSAGES); + assertThat(confOf(producer).getMaxPendingMessagesAcrossPartitions()) + .isEqualTo(NO_MEMORY_LIMIT_MAX_PENDING_MESSAGES_ACROSS_PARTITIONS); + } + + /** + * The fallback is a default, not a floor. An application that asks for no message-count limit at + * all still gets it, by passing 0 explicitly. This is what keeps 0 a usable value rather than an + * alias for "unset". + * + *

A single {@code maxPendingMessages(0)} has to be enough whatever the topic's shape: filling + * in the across-partitions budget would put a per-partition limit back on a partitioned topic. + */ + @Test(dataProvider = "partitioned") + public void testExplicitZeroDisablesTheBoundWhenMemoryLimitDisabled(boolean partitioned) throws Exception { + String topic = newTopicName(); + if (partitioned) { + admin.topics().createPartitionedTopic(topic, 10); + } + + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(brokerUrl.toString()) + .memoryLimit(0, SizeUnit.BYTES) + .build(); + + @Cleanup + Producer producer = client.newProducer(Schema.BYTES) + .topic(topic) + .maxPendingMessages(0) + .create(); + + assertThat(confOf(producer).getMaxPendingMessages()).isZero(); + assertThat(confOf(producer).getMaxPendingMessagesAcrossPartitions()).isZero(); + } + + /** + * Mirror of the above: disabling only the across-partitions budget must not take the per-producer + * default down with it. Filling in that default would otherwise be capped by a budget of 0. + */ + @SuppressWarnings("deprecation") + @Test + public void testExplicitZeroAcrossPartitionsKeepsThePerProducerDefault() throws Exception { + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(brokerUrl.toString()) + .memoryLimit(0, SizeUnit.BYTES) + .build(); + + @Cleanup + Producer producer = client.newProducer() + .topic(newTopicName()) + .maxPendingMessagesAcrossPartitions(0) + .create(); + + assertThat(confOf(producer).getMaxPendingMessages()) + .isEqualTo(NO_MEMORY_LIMIT_MAX_PENDING_MESSAGES); + assertThat(confOf(producer).getMaxPendingMessagesAcrossPartitions()).isZero(); + } + + /** + * {@code loadConf} is the other way an application configures a limit. A limit present in the map + * counts as configured, including a 0, even though {@code loadConf} rebuilds the configuration + * object and so cannot carry any marker on it. + */ + @Test + public void testLoadConfZeroDisablesTheBoundWhenMemoryLimitDisabled() throws Exception { + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(brokerUrl.toString()) + .memoryLimit(0, SizeUnit.BYTES) + .build(); + + @Cleanup + Producer producer = client.newProducer() + .topic(newTopicName()) + .loadConf(Map.of("maxPendingMessages", 0)) + .create(); + + assertThat(confOf(producer).getMaxPendingMessages()).isZero(); + assertThat(confOf(producer).getMaxPendingMessagesAcrossPartitions()).isZero(); + } + + /** + * A {@code loadConf} that does not mention the limits leaves them unconfigured, so the defaults + * still apply. Pins that rebuilding the configuration is not mistaken for configuring it. + */ + @Test + public void testLoadConfWithoutTheLimitsKeepsTheDefaults() throws Exception { + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(brokerUrl.toString()) + .memoryLimit(0, SizeUnit.BYTES) + .build(); + + @Cleanup + Producer producer = client.newProducer() + .topic(newTopicName()) + .loadConf(Map.of("producerName", "loadConfWithoutLimits")) + .create(); + + assertThat(confOf(producer).getMaxPendingMessages()) + .isEqualTo(NO_MEMORY_LIMIT_MAX_PENDING_MESSAGES); + assertThat(confOf(producer).getMaxPendingMessagesAcrossPartitions()) + .isEqualTo(NO_MEMORY_LIMIT_MAX_PENDING_MESSAGES_ACROSS_PARTITIONS); + } + + /** + * A cloned builder has to keep knowing which limits were configured, or the clone would silently + * get the defaults back. + */ + @Test + public void testCloneKeepsAnExplicitlyDisabledBound() throws Exception { + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(brokerUrl.toString()) + .memoryLimit(0, SizeUnit.BYTES) + .build(); + + ProducerBuilder builder = client.newProducer().maxPendingMessages(0); + + @Cleanup + Producer producer = builder.clone().topic(newTopicName()).create(); + + assertThat(confOf(producer).getMaxPendingMessages()).isZero(); + } + + /** + * {@code maxPendingMessagesAcrossPartitions} must be {@code >= maxPendingMessages}. Filling in + * the across-partitions fallback must therefore never lower it below an explicitly configured + * per-partition limit, which would fail producer creation. + */ + @Test + public void testExplicitMaxPendingMessagesAboveTheFallbackDoesNotFailCreation() throws Exception { + int maxPendingMessages = NO_MEMORY_LIMIT_MAX_PENDING_MESSAGES_ACROSS_PARTITIONS + 10_000; + + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(brokerUrl.toString()) + .memoryLimit(0, SizeUnit.BYTES) + .build(); + + @Cleanup + Producer producer = client.newProducer() + .topic(newTopicName()) + .maxPendingMessages(maxPendingMessages) + .create(); + + assertThat(confOf(producer).getMaxPendingMessages()).isEqualTo(maxPendingMessages); + assertThat(confOf(producer).getMaxPendingMessagesAcrossPartitions()) + .isGreaterThanOrEqualTo(maxPendingMessages); + } + + /** + * Filling in the fallback must not write it back into the builder's own configuration. The + * builder stays reusable, and a limit set on it afterwards is still validated against what the + * caller configured rather than against a filled-in default. + */ + @SuppressWarnings("deprecation") + @Test + public void testFallbackLeavesTheBuilderReusable() throws Exception { + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(brokerUrl.toString()) + .memoryLimit(0, SizeUnit.BYTES) + .build(); + + ProducerBuilder builder = client.newProducer(); + + @Cleanup + Producer first = builder.topic(newTopicName()).create(); + assertThat(confOf(first).getMaxPendingMessages()) + .isEqualTo(NO_MEMORY_LIMIT_MAX_PENDING_MESSAGES); + + // Rejected if creating the first producer had left the fallback in the builder, since the + // across-partitions limit has to be >= maxPendingMessages. + @Cleanup + Producer second = builder.topic(newTopicName()) + .maxPendingMessagesAcrossPartitions(500) + .create(); + assertThat(confOf(second).getMaxPendingMessages()).isEqualTo(500); + } + + /** + * The fallback has to reach partitioned producers too, where the per-partition queue is derived + * from the across-partitions budget. + */ + @Test + public void testPartitionedProducerIsBoundedWhenMemoryLimitDisabled() throws Exception { + String topic = newTopicName(); + admin.topics().createPartitionedTopic(topic, 10); + + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(brokerUrl.toString()) + .memoryLimit(0, SizeUnit.BYTES) + .build(); + + @Cleanup + Producer producer = client.newProducer().topic(topic).create(); + + // The budget spread over 10 partitions is well above the per-producer default, so each + // partition keeps the full default. + assertThat(confOf(producer).getMaxPendingMessages()) + .isEqualTo(NO_MEMORY_LIMIT_MAX_PENDING_MESSAGES); + assertThat(confOf(producer).getMaxPendingMessagesAcrossPartitions()) + .isEqualTo(NO_MEMORY_LIMIT_MAX_PENDING_MESSAGES_ACROSS_PARTITIONS); + } + + /** + * The across-partitions limit is a budget shared by every partition, so the per-producer + * fallback must be capped by it. Otherwise the fallback would exceed an explicitly configured + * budget, which producer creation rejects. + */ + @SuppressWarnings("deprecation") + @Test + public void testExplicitAcrossPartitionsLimitCapsTheFallback() throws Exception { + int maxPendingMessagesAcrossPartitions = 500; + + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(brokerUrl.toString()) + .memoryLimit(0, SizeUnit.BYTES) + .build(); + + @Cleanup + Producer producer = client.newProducer() + .topic(newTopicName()) + .maxPendingMessagesAcrossPartitions(maxPendingMessagesAcrossPartitions) + .create(); + + assertThat(confOf(producer).getMaxPendingMessages()) + .isEqualTo(maxPendingMessagesAcrossPartitions); + assertThat(confOf(producer).getMaxPendingMessagesAcrossPartitions()) + .isEqualTo(maxPendingMessagesAcrossPartitions); + } + + /** + * The fallback only exists to replace the missing byte-based backpressure. When a memory limit + * is configured, an unset pending-message limit keeps meaning "no message-count limit". + */ + @Test + public void testMemoryLimitedClientKeepsUnboundedPendingMessages() throws Exception { + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(brokerUrl.toString()) + .memoryLimit(64, SizeUnit.MEGA_BYTES) + .build(); + + @Cleanup + Producer producer = client.newProducer().topic(newTopicName()).create(); + + assertThat(confOf(producer).getMaxPendingMessages()).isZero(); + assertThat(confOf(producer).getMaxPendingMessagesAcrossPartitions()).isZero(); + } } diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java index 6015bc7af2174..ec9d93dbfea7d 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java @@ -174,7 +174,15 @@ public interface ProducerBuilder extends Cloneable { * the client application. Until the producer gets a successful acknowledgment back from the broker, * it will keep in memory (direct memory pool) all the messages in the pending queue. * - *

Default is 0, which disables the pending messages check. + *

Default is 0, which disables the pending messages check. Disabling it only removes the + * message-count limit; the memory the pending queue may hold is then bounded by the client + * memory limit ({@link ClientBuilder#memoryLimit(long, SizeUnit)}) instead. + * + *

On a client whose memory limit is disabled there would be no backpressure left at all, so a + * producer that does not configure this setting falls back to a default queue size of 1000 rather + * than buffering without limit. Calling this method always wins over that default, so passing 0 + * explicitly is how an application asks for a producer with no message-count limit, on a + * partitioned topic as well. * * @param maxPendingMessages * the max size of the pending messages queue for the producer @@ -190,7 +198,10 @@ public interface ProducerBuilder extends Cloneable { * The purpose of this setting is to have an upper-limit on the number * of pending messages when publishing on a partitioned topic. * - *

Default is 0, which disables the pending messages across partitions check. + *

Default is 0, which disables the pending messages across partitions check. As with + * {@link #maxPendingMessages(int)}, a producer that does not configure this setting on a client + * whose memory limit is disabled falls back to a default budget of 50000 instead, since no + * backpressure would otherwise be left, and calling this method always wins over that default. * *

If publishing at a high rate over a topic with many partitions (especially when publishing messages without a * partitioning key), it might be beneficial to increase this parameter to allow for more pipelining within the diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java index 3457aac0402fa..6d6c2081e2dda 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java @@ -56,6 +56,14 @@ public class ProducerBuilderImpl implements ProducerBuilder { private ProducerConfigurationData conf; private Schema schema; private List interceptorList; + /** + * Whether the application configured the pending-message limits. Their unset value is 0, which is + * also a meaningful explicit value ("no message-count limit"), so the configuration alone cannot + * tell the two apart. See + * {@link PulsarClientImpl#applyNoMemoryLimitProducerDefaults(ProducerConfigurationData, boolean, boolean)}. + */ + private boolean maxPendingMessagesConfigured; + private boolean maxPendingMessagesAcrossPartitionsConfigured; public ProducerBuilderImpl(PulsarClientImpl client, Schema schema) { this(client, new ProducerConfigurationData(), schema); @@ -78,7 +86,10 @@ public ProducerBuilder schema(Schema schema) { @Override public ProducerBuilder clone() { - return new ProducerBuilderImpl<>(client, conf.clone(), schema); + ProducerBuilderImpl copy = new ProducerBuilderImpl<>(client, conf.clone(), schema); + copy.maxPendingMessagesConfigured = maxPendingMessagesConfigured; + copy.maxPendingMessagesAcrossPartitionsConfigured = maxPendingMessagesAcrossPartitionsConfigured; + return copy; } @Override @@ -108,15 +119,23 @@ public CompletableFuture> createAsync() { return FutureUtil.failedFuture(pce); } + ProducerConfigurationData producerConf = client.applyNoMemoryLimitProducerDefaults(conf, + maxPendingMessagesConfigured, maxPendingMessagesAcrossPartitionsConfigured); + return interceptorList == null || interceptorList.size() == 0 - ? client.createProducerAsync(conf, schema, null) - : client.createProducerAsync(conf, schema, new ProducerInterceptors(interceptorList)); + ? client.createProducerAsync(producerConf, schema, null) + : client.createProducerAsync(producerConf, schema, new ProducerInterceptors(interceptorList)); } @Override public ProducerBuilder loadConf(Map config) { conf = ConfigurationDataUtils.loadData( config, conf, ProducerConfigurationData.class); + // A limit present in the map was configured by the application, even when its value is the + // same as the unset one. loadData builds a new configuration instance, so this cannot be + // tracked in the configuration itself. + maxPendingMessagesConfigured |= config.containsKey("maxPendingMessages"); + maxPendingMessagesAcrossPartitionsConfigured |= config.containsKey("maxPendingMessagesAcrossPartitions"); return this; } @@ -142,6 +161,7 @@ public ProducerBuilder sendTimeout(int sendTimeout, @NonNull TimeUnit unit) { @Override public ProducerBuilder maxPendingMessages(int maxPendingMessages) { conf.setMaxPendingMessages(maxPendingMessages); + maxPendingMessagesConfigured = true; return this; } @@ -149,6 +169,7 @@ public ProducerBuilder maxPendingMessages(int maxPendingMessages) { @Override public ProducerBuilder maxPendingMessagesAcrossPartitions(int maxPendingMessagesAcrossPartitions) { conf.setMaxPendingMessagesAcrossPartitions(maxPendingMessagesAcrossPartitions); + maxPendingMessagesAcrossPartitionsConfigured = true; return this; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java index 5f9acf2ff313c..fd626c24d4715 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java @@ -305,14 +305,7 @@ public ProducerBuilder newProducer() { @Override public ProducerBuilder newProducer(Schema schema) { - ProducerBuilderImpl producerBuilder = new ProducerBuilderImpl<>(this, schema); - if (!memoryLimitController.isMemoryLimited()) { - // set default limits for producers when memory limit controller is disabled - producerBuilder.maxPendingMessages(NO_MEMORY_LIMIT_DEFAULT_MAX_PENDING_MESSAGES); - producerBuilder.maxPendingMessagesAcrossPartitions( - NO_MEMORY_LIMIT_DEFAULT_MAX_PENDING_MESSAGES_ACROSS_PARTITIONS); - } - return producerBuilder; + return new ProducerBuilderImpl<>(this, schema); } @Override @@ -401,6 +394,78 @@ public CompletableFuture> createProducerAsync(ProducerConfigurat } + /** + * Apply the default pending-message limits a producer gets when this client has no memory limit. + * + *

The client memory limit is a producer's primary backpressure: it bounds the memory held by + * messages that have been queued but not yet acknowledged by the broker. When it is disabled + * there is nothing left to bound that queue, so producers fall back to the pre-PIP-120 + * message-count defaults rather than buffering without any limit at all. + * + *

These are defaults, not a floor. A limit the application configured is always kept — including + * an explicit {@code 0}, which is how an application asks for no message-count limit at all. Only a + * limit that was never configured is filled in, which is why the caller passes in what it saw + * rather than letting this method infer it: {@code 0} is both the unset value and a meaningful + * explicit one. + * + *

Note that on a partitioned topic a filled-in across-partitions budget is still divided between + * the partitions afterwards, which can lower an explicitly configured per-producer limit. + * + *

Called by {@link ProducerBuilderImpl}, which is what knows whether a limit was configured. + * + * @param conf the requested producer configuration + * @param maxPendingMessagesConfigured whether the application configured {@code maxPendingMessages} + * @param maxPendingMessagesAcrossPartitionsConfigured whether the application configured + * {@code maxPendingMessagesAcrossPartitions} + * @return the configuration to create the producer with; a resolved copy when a default applies, + * otherwise {@code conf} unchanged + */ + public ProducerConfigurationData applyNoMemoryLimitProducerDefaults(ProducerConfigurationData conf, + boolean maxPendingMessagesConfigured, boolean maxPendingMessagesAcrossPartitionsConfigured) { + // A limit that is already positive was configured by definition, whichever way the + // configuration was populated. The flags only tell an explicit 0 apart from an unset one. + maxPendingMessagesConfigured |= conf.getMaxPendingMessages() > 0; + maxPendingMessagesAcrossPartitionsConfigured |= conf.getMaxPendingMessagesAcrossPartitions() > 0; + if ((maxPendingMessagesConfigured && maxPendingMessagesAcrossPartitionsConfigured) + || memoryLimitController.isMemoryLimited()) { + return conf; + } + int maxPendingMessages = maxPendingMessagesConfigured + ? conf.getMaxPendingMessages() + : NO_MEMORY_LIMIT_DEFAULT_MAX_PENDING_MESSAGES; + final int maxPendingMessagesAcrossPartitions; + if (maxPendingMessagesAcrossPartitionsConfigured) { + maxPendingMessagesAcrossPartitions = conf.getMaxPendingMessagesAcrossPartitions(); + } else if (maxPendingMessages == 0) { + // The application configured no per-producer limit. Filling in a partitions budget would + // put one back, because a partitioned producer derives its per-partition limit from it, so + // a single maxPendingMessages(0) is enough to ask for a producer with no message-count + // limit whatever the topic's shape. + maxPendingMessagesAcrossPartitions = 0; + } else { + maxPendingMessagesAcrossPartitions = + Math.max(maxPendingMessages, NO_MEMORY_LIMIT_DEFAULT_MAX_PENDING_MESSAGES_ACROSS_PARTITIONS); + } + if (maxPendingMessagesAcrossPartitions > 0) { + // The across-partitions limit is a budget shared by every partition, so a single producer's + // queue can never exceed it. A configured 0 means there is no such budget and is left + // alone, rather than capping every producer at zero. + maxPendingMessages = Math.min(maxPendingMessages, maxPendingMessagesAcrossPartitions); + } + + // Resolve on a copy: the builder hands over its own configuration instance, so filling in a + // limit here would otherwise leak into the next producer built from the same builder. + ProducerConfigurationData resolved = conf.clone(); + resolved.setMaxPendingMessages(maxPendingMessages); + resolved.setMaxPendingMessagesAcrossPartitions(maxPendingMessagesAcrossPartitions); + if (log.isDebugEnabled()) { + log.debug("[{}] Client memory limit is disabled, applying default producer pending message limits." + + " maxPendingMessages: {}, maxPendingMessagesAcrossPartitions: {}", + conf.getTopicName(), maxPendingMessages, maxPendingMessagesAcrossPartitions); + } + return resolved; + } + public CompletableFuture reloadSchemaForAutoProduceProducer(String topic, AutoProduceBytesSchema autoSchema) { return lookup.getSchema(TopicName.get(topic)).thenAccept(schemaInfoOptional -> { if (schemaInfoOptional.isPresent()) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java index a7da82599398d..cec15904f71a6 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java @@ -250,9 +250,17 @@ public void setMaxPendingMessages(int maxPendingMessages) { this.maxPendingMessages = maxPendingMessages; } + /** + * The across-partitions budget used to be rejected when it was below {@link #maxPendingMessages}, + * which made the two setters order-dependent: it depended on which of them had been called first, + * and it made {@code loadConf} fail outright for any positive {@code maxPendingMessages}, since + * that replays every property through the setters in an order the caller does not control. The + * relationship is enforced where it is used instead — {@code PartitionedProducerImpl} lowers the + * per-partition limit to the share of the budget when a budget is set. + */ public void setMaxPendingMessagesAcrossPartitions(int maxPendingMessagesAcrossPartitions) { - checkArgument(maxPendingMessagesAcrossPartitions >= maxPendingMessages, - "maxPendingMessagesAcrossPartitions needs to be >= maxPendingMessages"); + checkArgument(maxPendingMessagesAcrossPartitions >= 0, + "maxPendingMessagesAcrossPartitions needs to be >= 0"); this.maxPendingMessagesAcrossPartitions = maxPendingMessagesAcrossPartitions; } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java index a49114d67b58f..6033ebc1146e4 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java @@ -19,12 +19,16 @@ package org.apache.pulsar.client.impl; import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyBoolean; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -60,6 +64,11 @@ public void setup() { producerBuilderImpl = new ProducerBuilderImpl<>(client, Schema.BYTES); when(client.newProducer()).thenReturn(producerBuilderImpl); + // The builder asks the client to fill in the pending-message defaults before creating the + // producer; on a mock that would otherwise hand back a null configuration. + when(client.applyNoMemoryLimitProducerDefaults(any(ProducerConfigurationData.class), anyBoolean(), + anyBoolean())).thenAnswer(invocation -> invocation.getArgument(0)); + when(client.createProducerAsync( any(ProducerConfigurationData.class), any(Schema.class), eq(null))) .thenReturn(CompletableFuture.completedFuture(producer)); @@ -116,6 +125,23 @@ public void testProducerBuilderImplWhenMessageRoutingModeIsRoundRobinPartition() assertNotNull(producer); } + /** + * {@code loadConf} rebuilds the configuration by replaying every property through the public + * setters, and {@code setMaxPendingMessagesAcrossPartitions} rejects a value below + * {@code maxPendingMessages}. Pins that loading a positive limit does not trip that check on the + * across-partitions property that comes with it, and that the limit is recorded as configured. + */ + @SuppressWarnings("deprecation") + @Test + public void testLoadConfWithAPositiveMaxPendingMessages() { + producerBuilderImpl = new ProducerBuilderImpl<>(client, Schema.BYTES); + producerBuilderImpl.loadConf(Collections.singletonMap("maxPendingMessages", 5000)); + + assertEquals(producerBuilderImpl.getConf().getMaxPendingMessages(), 5000); + assertTrue(producerBuilderImpl.isMaxPendingMessagesConfigured()); + assertFalse(producerBuilderImpl.isMaxPendingMessagesAcrossPartitionsConfigured()); + } + @Test public void testProducerBuilderImplWhenMessageRoutingIsSetImplicitly() throws PulsarClientException { producerBuilderImpl = new ProducerBuilderImpl(client, Schema.BYTES); @@ -373,11 +399,25 @@ public void testProducerBuilderImplWhenMaxPendingMessagesAcrossPartitionsPropert } @Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = - "maxPendingMessagesAcrossPartitions needs to be >= maxPendingMessages") + "maxPendingMessagesAcrossPartitions needs to be >= 0") public void testProducerBuilderImplWhenMaxPendingMessagesAcrossPartitionsPropertyIsInvalidErrorMessages() { producerBuilderImpl.maxPendingMessagesAcrossPartitions(-1); } + /** + * The across-partitions budget is allowed to be below {@code maxPendingMessages}: it is a budget + * shared by every partition, and the per-partition limit is lowered to its share where it is used. + * Rejecting it here made the two setters order-dependent. + */ + @SuppressWarnings("deprecation") + @Test + public void testAcrossPartitionsLimitBelowMaxPendingMessagesIsAccepted() { + producerBuilderImpl = new ProducerBuilderImpl<>(client, Schema.BYTES); + producerBuilderImpl.maxPendingMessages(1000).maxPendingMessagesAcrossPartitions(500); + + assertEquals(producerBuilderImpl.getConf().getMaxPendingMessagesAcrossPartitions(), 500); + } + @Test public void testProducerBuilderImplWhenNumericPropertiesAreValid() { producerBuilderImpl.batchingMaxPublishDelay(1, TimeUnit.SECONDS); diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/ContextImplTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/ContextImplTest.java index cb4c93f153fd9..071886698fb45 100644 --- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/ContextImplTest.java +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/ContextImplTest.java @@ -20,6 +20,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.same; @@ -108,6 +109,10 @@ public void setup() throws PulsarClientException { when(client.newProducer()).thenAnswer(invocation -> new ProducerBuilderImpl(client, Schema.BYTES)); when(client.newProducer(any())).thenAnswer( invocation -> new ProducerBuilderImpl(client, invocation.getArgument(0))); + // The builder asks the client to fill in the pending-message defaults before creating the + // producer; on a mock that would otherwise hand back a null configuration. + when(client.applyNoMemoryLimitProducerDefaults(any(ProducerConfigurationData.class), anyBoolean(), + anyBoolean())).thenAnswer(invocation -> invocation.getArgument(0)); when(client.createProducerAsync(any(ProducerConfigurationData.class), any(), any())) .thenReturn(CompletableFuture.completedFuture(producer)); when(client.getSchema(anyString())).thenReturn(CompletableFuture.completedFuture(Optional.empty())); From 691f8127cbc5539caa1e7fbf1bba2d570c363dc9 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 19 Aug 2026 12:05:43 +0300 Subject: [PATCH 179/213] [fix][cli][branch-4.0] Don't pass unset pulsar-perf pending-message options to the producer (#26371) Backport of the branch-4.2 fix, limited to pulsar-testclient: #26371's hunks in ProducerConfigurationData and ProducerBuilderImplTest already arrived on this branch with the #26342 backport (4c9e045854e), which carries the same "maxPendingMessagesAcrossPartitions needs to be >= 0" relaxation. The branch-4.2 follow-up d705ea3, which adapted the pulsar-perf test once #26342 landed there, is folded in here because on this branch #26342 landed first. (cherry picked from commit e0d4fd3d880b7b7f6fc9eb8be5b71d131644d19e) (cherry picked from commit d705ea36dd7ae86161b271108322635c503f3b0f) --- .../testclient/PerformanceProducer.java | 25 ++++++--- .../testclient/PerformanceProducerTest.java | 55 +++++++++++++++++++ 2 files changed, 71 insertions(+), 9 deletions(-) diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java index 8860696321a3e..5394838f8a512 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java @@ -22,8 +22,6 @@ import static java.util.concurrent.TimeUnit.NANOSECONDS; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.pulsar.client.impl.conf.ProducerConfigurationData.DEFAULT_BATCHING_MAX_MESSAGES; -import static org.apache.pulsar.client.impl.conf.ProducerConfigurationData.DEFAULT_MAX_PENDING_MESSAGES; -import static org.apache.pulsar.client.impl.conf.ProducerConfigurationData.DEFAULT_MAX_PENDING_MESSAGES_ACROSS_PARTITIONS; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.google.common.collect.Range; @@ -64,6 +62,7 @@ import org.apache.pulsar.client.api.ProducerBuilder; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.TypedMessageBuilder; import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; @@ -134,12 +133,13 @@ public class PerformanceProducer extends PerformanceTopicListArguments{ + "larger than allowed max size") private boolean chunkingAllowed = false; - @Option(names = { "-o", "--max-outstanding" }, description = "Max number of outstanding messages") - public int maxOutstanding = DEFAULT_MAX_PENDING_MESSAGES; + @Option(names = { "-o", "--max-outstanding" }, description = "Max number of outstanding messages. " + + "Left to the client default when unset") + public Integer maxOutstanding; @Option(names = { "-p", "--max-outstanding-across-partitions" }, description = "Max number of outstanding " - + "messages across partitions") - public int maxPendingMessagesAcrossPartitions = DEFAULT_MAX_PENDING_MESSAGES_ACROSS_PARTITIONS; + + "messages across partitions. Left to the client default when unset") + public Integer maxPendingMessagesAcrossPartitions; @Option(names = { "-np", "--partitions" }, description = "Create partitioned topics with the given number " + "of partitions, set 0 to not try to create the topic") @@ -447,14 +447,21 @@ static IMessageFormatter getMessageFormatter(String formatterClass) { } ProducerBuilder createProducerBuilder(PulsarClient client, int producerId) { - ProducerBuilder producerBuilder = client.newProducer() // + // pulsar-perf runs with the client memory limit disabled unless --memory-limit is given, so the + // client's pending-message defaults are the producer's only backpressure. They are applied to + // any producer whose limits are left unset, whichever newProducer() overload is used. + ProducerBuilder producerBuilder = client.newProducer(Schema.BYTES) // .sendTimeout(this.sendTimeout, TimeUnit.SECONDS) // .compressionType(this.compression) // - .maxPendingMessages(this.maxOutstanding) // .accessMode(this.producerAccessMode) // enable round robin message routing if it is a partitioned topic .messageRoutingMode(MessageRoutingMode.RoundRobinPartition); - if (this.maxPendingMessagesAcrossPartitions > 0) { + // Only pass a limit the user actually asked for. Their unset value is 0, which the client reads + // as "no message-count limit", so passing it through would leave the producer unbounded. + if (this.maxOutstanding != null) { + producerBuilder.maxPendingMessages(this.maxOutstanding); + } + if (this.maxPendingMessagesAcrossPartitions != null) { producerBuilder.maxPendingMessagesAcrossPartitions(this.maxPendingMessagesAcrossPartitions); } diff --git a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceProducerTest.java b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceProducerTest.java index 47815ddddf8b1..277f72bcafc03 100644 --- a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceProducerTest.java +++ b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceProducerTest.java @@ -32,9 +32,12 @@ import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.impl.ProducerBase; import org.apache.pulsar.client.impl.ProducerBuilderImpl; +import org.apache.pulsar.client.impl.conf.ProducerConfigurationData; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.awaitility.Awaitility; @@ -236,6 +239,58 @@ public void testMaxOutstanding() throws Exception { consumer.close(); } + /** + * pulsar-perf runs with the client memory limit disabled unless {@code --memory-limit} is given, so + * the producer's only backpressure is its pending-message queue. Leaving the options unset has to + * leave the client's own defaults in place; passing their unset value of 0 through would read as an + * explicit "no message-count limit" and leave the producer unbounded, which is what exhausts direct + * memory against a slow broker on a non-partitioned topic. + */ + @Test(timeOut = 20000) + public void testPendingMessageLimitsAreLeftToTheClientWhenUnset() throws Exception { + PerformanceProducer producer = new PerformanceProducer(); + producer.topics = List.of(testTopic + UUID.randomUUID()); + producer.serviceURL = pulsar.getBrokerServiceUrl(); + + @Cleanup + PulsarClient client = PerfClientUtils.createClientBuilderFromArguments(producer).build(); + ProducerBuilderImpl builder = + (ProducerBuilderImpl) producer.createProducerBuilder(client, 0); + + Assert.assertNull(producer.maxOutstanding); + Assert.assertNull(producer.maxPendingMessagesAcrossPartitions); + // pulsar-perf must not configure either limit, so that they stay the client's to decide. + Assert.assertFalse(builder.isMaxPendingMessagesConfigured()); + Assert.assertFalse(builder.isMaxPendingMessagesAcrossPartitionsConfigured()); + + // The client resolves its no-memory-limit defaults when the producer is created rather than on + // the builder, so the effective configuration is where they have to show up. + @Cleanup + Producer createdProducer = builder.topic(producer.topics.get(0)).create(); + ProducerConfigurationData conf = ((ProducerBase) createdProducer).getConfiguration(); + Assert.assertTrue(conf.getMaxPendingMessages() > 0); + Assert.assertTrue(conf.getMaxPendingMessagesAcrossPartitions() > 0); + } + + /** + * An option that is given still wins over the client default, and either one can be given on its + * own — the across-partitions budget is allowed to be below the per-producer limit. + */ + @Test(timeOut = 20000) + public void testGivenPendingMessageLimitsAreApplied() throws Exception { + PerformanceProducer producer = new PerformanceProducer(); + producer.topics = List.of(testTopic + UUID.randomUUID()); + producer.serviceURL = pulsar.getBrokerServiceUrl(); + producer.maxPendingMessagesAcrossPartitions = 500; + + @Cleanup + PulsarClient client = PerfClientUtils.createClientBuilderFromArguments(producer).build(); + ProducerBuilderImpl builder = + (ProducerBuilderImpl) producer.createProducerBuilder(client, 0); + + Assert.assertEquals(builder.getConf().getMaxPendingMessagesAcrossPartitions(), 500); + } + @Test public void testRangeConvert() { PerformanceProducer.RangeConvert rangeConvert = new PerformanceProducer.RangeConvert(); From 7a4b4d1b912ae027260160879e04f25325123c88 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 19 Aug 2026 16:14:17 +0300 Subject: [PATCH 180/213] [fix][sec][branch-4.2] Upgrade BouncyCastle to 1.85 and BouncyCastle FIPS to 2.0.2 to address CVEs (#26370) --- distribution/server/src/assemble/LICENSE.bin.txt | 6 +++--- distribution/shell/src/assemble/LICENSE.bin.txt | 6 +++--- pom.xml | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index fcef3c1858a79..89309a1a4f8ad 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -633,9 +633,9 @@ Creative Commons Attribution License Bouncy Castle License * Bouncy Castle -- ../licenses/LICENSE-bouncycastle.txt - - org.bouncycastle-bcpkix-jdk18on-1.84.jar - - org.bouncycastle-bcprov-jdk18on-1.84.jar - - org.bouncycastle-bcutil-jdk18on-1.84.jar + - org.bouncycastle-bcpkix-jdk18on-1.85.jar + - org.bouncycastle-bcprov-jdk18on-1.85.2.jar + - org.bouncycastle-bcutil-jdk18on-1.85.jar ------------------------ diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index e5a6e42fd22c4..5ee203142d095 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -473,9 +473,9 @@ Creative Commons Attribution License Bouncy Castle License * Bouncy Castle -- ../licenses/LICENSE-bouncycastle.txt - - bcpkix-jdk18on-1.84.jar - - bcprov-jdk18on-1.84.jar - - bcutil-jdk18on-1.84.jar + - bcpkix-jdk18on-1.85.jar + - bcprov-jdk18on-1.85.2.jar + - bcutil-jdk18on-1.85.jar ------------------------ diff --git a/pom.xml b/pom.xml index 3a23235e9e878..46a6607dab037 100644 --- a/pom.xml +++ b/pom.xml @@ -202,13 +202,13 @@ flexible messaging model and an intuitive client API. 4.5.0 2.26.1 - 1.84 - ${bouncycastle.version} + 1.85 + 1.85.2 ${bouncycastle.version} ${bouncycastle.version} - 2.0.11 - 2.0.6 - 2.0.1 + 2.0.12 + 2.0.7 + 2.0.2 2.18.10 8.5.16 0.10.2 From 3ff54792fbc4ef515e5265e211c5f98a32d03850 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 19 Aug 2026 16:28:36 +0300 Subject: [PATCH 181/213] [fix][test][branch-4.2] Create the CLI test producer only once CmdProduce asks for it ### Motivation `PulsarClientToolTest.testDisableBatching` fails since the #26342 backport: the messages produced with `-db`/`--disable-batching` arrive as `BatchMessageIdImpl`, so batching was never actually disabled. `PulsarClientToolForceBatchNum` built the producer up front in `mockClientBuilder()` and stubbed `ProducerBuilder.create()` to hand back that one instance. `CmdProduce` applies `--disable-batching` by calling `producerBuilder.enableBatching(false)` afterwards, which only ever reached the already-created producer because that producer shared the builder's `ProducerConfigurationData` instance. #26342 resolves the no-memory-limit pending-message defaults onto a copy of the configuration when the producer is created, and `PulsarClientTool` leaves the client memory limit disabled, so that copy is what the producer now holds. A builder change made after creation no longer reaches it, and the option was silently dropped. ### Modifications Create the producer inside the stubbed `create()` rather than ahead of it, so that the configuration `CmdProduce` set up is the configuration the producer is created with. master's `PulsarClientToolForceBatchNum` already works this way after the v5 client API migration, which is why the test only fails here; branch-4.2 stays on the v4 API, so the same laziness is applied to it. The `send()` to `sendAsync()` wrapping moves into a `forceAsyncSend()` helper and is otherwise unchanged. Test-only. The test discriminates in both directions: the same helper yields batched message ids without `-db` and non-batched ids with it. --- .../cli/PulsarClientToolForceBatchNum.java | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolForceBatchNum.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolForceBatchNum.java index 896bee0e030af..a5d3ddc22a35a 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolForceBatchNum.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolForceBatchNum.java @@ -39,7 +39,7 @@ * An implement of {@link PulsarClientTool} for test, which will publish messages iff there is enough messages * in the batch. */ -public class PulsarClientToolForceBatchNum extends PulsarClientTool{ +public class PulsarClientToolForceBatchNum extends PulsarClientTool { private final String topic; private final int batchNum; @@ -68,18 +68,35 @@ public void updateConfig(ClientBuilder newBuilder, Authentication authentication private ClientBuilder mockClientBuilder(ClientBuilder newBuilder) throws Exception { PulsarClientImpl client = (PulsarClientImpl) newBuilder.build(); + // Batch exactly batchNum messages and (practically) never flush on the timer, so that batching is + // deterministic. CmdProduce leaves batching alone unless it was asked to disable it. ProducerBuilder producerBuilder = client.newProducer() .batchingMaxBytes(Integer.MAX_VALUE) .batchingMaxMessages(batchNum) .batchingMaxPublishDelay(Long.MAX_VALUE, TimeUnit.MILLISECONDS) .topic(topic); - Producer producer = producerBuilder.create(); PulsarClientImpl mockClient = spy(client); ProducerBuilder mockProducerBuilder = spy(producerBuilder); - Producer mockProducer = spy(producer); ClientBuilder mockClientBuilder = spy(newBuilder); + // Create the producer only once CmdProduce asks for it. It configures the builder it was handed + // first, and options such as --disable-batching have to reach the producer that is created from + // it: a producer holds the configuration it was created with, so a builder change made after + // creation no longer affects it. + doAnswer(invocation -> forceAsyncSend((Producer) invocation.callRealMethod())) + .when(mockProducerBuilder).create(); + doReturn(mockProducerBuilder).when(mockClient).newProducer(any(Schema.class)); + doReturn(mockClient).when(mockClientBuilder).build(); + return mockClientBuilder; + } + + /** + * Wraps the producer so that the synchronous {@code newMessage().send()} the CLI uses is dispatched + * asynchronously, letting messages accumulate into a batch instead of flushing one by one. + */ + private static Producer forceAsyncSend(Producer producer) { + Producer mockProducer = spy(producer); doAnswer((Answer) invocation -> { TypedMessageBuilder typedMessageBuilder = spy((TypedMessageBuilder) invocation.callRealMethod()); doAnswer((Answer) invocation1 -> { @@ -90,10 +107,6 @@ private ClientBuilder mockClientBuilder(ClientBuilder newBuilder) throws Excepti }).when(typedMessageBuilder).send(); return typedMessageBuilder; }).when(mockProducer).newMessage(); - - doReturn(mockProducer).when(mockProducerBuilder).create(); - doReturn(mockProducerBuilder).when(mockClient).newProducer(any(Schema.class)); - doReturn(mockClient).when(mockClientBuilder).build(); - return mockClientBuilder; + return mockProducer; } } From 55f21904c0edf2f4161a29e2eca037bab93da27c Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 19 Aug 2026 11:35:45 +0300 Subject: [PATCH 182/213] [fix][cli] Set --enable-native-access=ALL-UNNAMED for the CLI tools on Java 24+ (#26365) (cherry picked from commit 68aa11c2a46eb846feea1584c35ab38f0d1669db) --- bin/pulsar-admin-common.sh | 8 ++++++++ bin/pulsar-perf | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/bin/pulsar-admin-common.sh b/bin/pulsar-admin-common.sh index 366d76d7f5b98..ca341b3b5fc45 100755 --- a/bin/pulsar-admin-common.sh +++ b/bin/pulsar-admin-common.sh @@ -127,6 +127,14 @@ if [[ $JAVA_MAJOR_VERSION -ge 11 ]]; then OPTS="$OPTS --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED" fi +if [[ $JAVA_MAJOR_VERSION -ge 24 ]]; then + # Netty loads native libraries (epoll, io_uring, tcnative) via java.lang.System::loadLibrary, + # which is a restricted method from Java 24 onwards. Without this the JVM prints a warning to + # stderr on every invocation, and restricted methods will be blocked outright in a future + # release. bin/pulsar already sets this for the server side. + OPTS="$OPTS --enable-native-access=ALL-UNNAMED" +fi + OPTS="-cp $PULSAR_CLASSPATH $OPTS" OPTS="$OPTS $PULSAR_EXTRA_OPTS" diff --git a/bin/pulsar-perf b/bin/pulsar-perf index 2b120cc2f94af..0c61aa9de503a 100755 --- a/bin/pulsar-perf +++ b/bin/pulsar-perf @@ -124,6 +124,14 @@ if [[ $JAVA_MAJOR_VERSION -ge 11 ]]; then OPTS="$OPTS --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED" fi +if [[ $JAVA_MAJOR_VERSION -ge 24 ]]; then + # Netty loads native libraries (epoll, io_uring, tcnative) via java.lang.System::loadLibrary, + # which is a restricted method from Java 24 onwards. Without this the JVM prints a warning to + # stderr on every invocation, and restricted methods will be blocked outright in a future + # release. bin/pulsar already sets this for the server side. + OPTS="$OPTS --enable-native-access=ALL-UNNAMED" +fi + OPTS="-cp $PULSAR_CLASSPATH $OPTS" OPTS="$OPTS $PULSAR_EXTRA_OPTS" From db9452e2eaa0a87483caa57b7f16cc341f7a9583 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 19 Aug 2026 12:01:44 +0300 Subject: [PATCH 183/213] [fix][test] Fix OneWayReplicatorSchemaValidationEnforcedTest racing the replicator's remote topic creation (#26382) (cherry picked from commit bbc02f0867f742f386949c911b2a3be7c03be62b) --- .../OneWayReplicatorSchemaValidationEnforcedTest.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorSchemaValidationEnforcedTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorSchemaValidationEnforcedTest.java index ed8f8ac479ddd..27579c1ea2fda 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorSchemaValidationEnforcedTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorSchemaValidationEnforcedTest.java @@ -70,12 +70,15 @@ public void testReplicationWithAvroSchemaWithSchemaValidationEnforced() throws E Schema myClassSchema = Schema.AVRO(MyClass.class); final String topicName = BrokerTestUtil.newUniqueName("persistent://" + sourceClusterAlwaysSchemaCompatibleNamespace + "/tp_"); + // Create the topic and schema in the remote cluster (r2) first. Creating the topic in the local + // cluster starts the replicator, which creates the topic on the remote cluster itself when it is + // missing (GeoPersistentReplicator#createRemoteTopicIfDoesNotExist), so creating r2's topic + // afterwards races with it and fails with "This topic already exists". + admin2.topics().createNonPartitionedTopic(topicName); + admin2.schemas().createSchema(topicName, myClassSchema.getSchemaInfo()); // create the topic and schema in the local cluster (r1) admin1.topics().createNonPartitionedTopic(topicName); admin1.schemas().createSchema(topicName, myClassSchema.getSchemaInfo()); - // create the topic and schema in the remote cluster (r2) - admin2.topics().createNonPartitionedTopic(topicName); - admin2.schemas().createSchema(topicName, myClassSchema.getSchemaInfo()); // consume from the remote cluster (r2) org.apache.pulsar.client.api.Consumer consumer2 = client2.newConsumer(myClassSchema) From ebad291137aadf0972b0868bfa83a2770f5fa0cc Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 19 Aug 2026 18:17:58 +0300 Subject: [PATCH 184/213] [fix][client] Divide the across-partitions budget only when it was set explicitly (#26384) (cherry picked from commit 3be684acf3d81c4ff34bbd6dcd4c2cb41539f9b3) --- .../client/api/ProducerQueueSizeTest.java | 103 ++++++++++++++++++ .../client/impl/PartitionedProducerImpl.java | 21 ++-- .../client/impl/ProducerBuilderImpl.java | 34 +++--- .../pulsar/client/impl/PulsarClientImpl.java | 43 ++++---- .../impl/conf/ProducerConfigurationData.java | 26 +++++ .../impl/PartitionedProducerImplTest.java | 11 +- .../client/impl/ProducerBuilderImplTest.java | 9 +- .../functions/instance/ContextImplTest.java | 5 +- .../testclient/PerformanceProducerTest.java | 4 +- 9 files changed, 197 insertions(+), 59 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerQueueSizeTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerQueueSizeTest.java index 33badf3e52fe6..42272a6951263 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerQueueSizeTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ProducerQueueSizeTest.java @@ -379,4 +379,107 @@ public void testMemoryLimitedClientKeepsUnboundedPendingMessages() throws Except assertThat(confOf(producer).getMaxPendingMessages()).isZero(); assertThat(confOf(producer).getMaxPendingMessagesAcrossPartitions()).isZero(); } + + /** + * A budget the application asked for is what gets divided between the partitions. Pins that the + * per-producer default filled in alongside it does not win the division and cap every partition at + * that default instead of at its share of the budget. + */ + @SuppressWarnings("deprecation") + @Test + public void testExplicitAcrossPartitionsBudgetIsDividedBetweenThePartitions() throws Exception { + String topic = newTopicName(); + admin.topics().createPartitionedTopic(topic, 10); + + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(lookupUrl.toString()) + .memoryLimit(0, SizeUnit.BYTES) + .build(); + + @Cleanup + Producer producer = client.newProducer() + .topic(topic) + .maxPendingMessagesAcrossPartitions(60_000) + .create(); + + assertThat(confOf(producer).getMaxPendingMessages()).isEqualTo(6_000); + } + + /** + * The mirror image: a budget that was only filled in as a default must not be divided, because + * dividing it would lower a per-producer limit the application did ask for. + */ + @Test + public void testFilledInBudgetDoesNotLowerAnExplicitPerProducerLimit() throws Exception { + String topic = newTopicName(); + admin.topics().createPartitionedTopic(topic, 10); + + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(lookupUrl.toString()) + .memoryLimit(0, SizeUnit.BYTES) + .build(); + + @Cleanup + Producer producer = client.newProducer() + .topic(topic) + .maxPendingMessages(60_000) + .create(); + + assertThat(confOf(producer).getMaxPendingMessages()).isEqualTo(60_000); + } + + /** + * A budget smaller than the partition count divides to zero, which used to remove the queue bound + * altogether — asking for a tighter budget made the producer unbounded. Each partition keeps the + * smallest possible queue instead. + */ + @SuppressWarnings("deprecation") + @Test + public void testBudgetSmallerThanThePartitionCountStillBoundsEachPartition() throws Exception { + String topic = newTopicName(); + admin.topics().createPartitionedTopic(topic, 10); + + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(lookupUrl.toString()) + .memoryLimit(0, SizeUnit.BYTES) + .build(); + + @Cleanup + Producer producer = client.newProducer() + .topic(topic) + .maxPendingMessagesAcrossPartitions(5) + .create(); + + assertThat(confOf(producer).getMaxPendingMessages()).isEqualTo(1); + } + + /** + * An explicit {@code 0} means "no message-count limit" and stays that way on a partitioned topic, + * even next to an across-partitions budget. Reading the unset value instead of the marker used to + * overwrite it with the budget's per-partition share. + */ + @SuppressWarnings("deprecation") + @Test + public void testExplicitZeroIsKeptAlongsideAnAcrossPartitionsBudget() throws Exception { + String topic = newTopicName(); + admin.topics().createPartitionedTopic(topic, 10); + + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(lookupUrl.toString()) + .memoryLimit(0, SizeUnit.BYTES) + .build(); + + @Cleanup + Producer producer = client.newProducer() + .topic(topic) + .maxPendingMessages(0) + .maxPendingMessagesAcrossPartitions(60_000) + .create(); + + assertThat(confOf(producer).getMaxPendingMessages()).isZero(); + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java index cca1899851507..dfc020ccb5721 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PartitionedProducerImpl.java @@ -19,8 +19,6 @@ package org.apache.pulsar.client.impl; import static com.google.common.base.Preconditions.checkArgument; -import static org.apache.pulsar.client.impl.conf.ProducerConfigurationData.DEFAULT_MAX_PENDING_MESSAGES; -import static org.apache.pulsar.client.impl.conf.ProducerConfigurationData.DEFAULT_MAX_PENDING_MESSAGES_ACROSS_PARTITIONS; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import io.netty.util.Timeout; @@ -85,14 +83,23 @@ public PartitionedProducerImpl(PulsarClientImpl client, String topic, ProducerCo ? new PartitionedTopicProducerStatsRecorderImpl() : null; + // The across-partitions budget is a total shared by every partition, so it is divided between + // them here, where the partition count is finally known. Both limits are read through their + // "configured" markers rather than by comparing against their unset value, because that value + // is 0 for both and 0 is also a meaningful explicit setting. // MaxPendingMessagesAcrossPartitions doesn't support partial partition such as SinglePartition correctly int maxPendingMessages = conf.getMaxPendingMessages(); int maxPendingMessagesAcrossPartitions = conf.getMaxPendingMessagesAcrossPartitions(); - if (maxPendingMessagesAcrossPartitions != DEFAULT_MAX_PENDING_MESSAGES_ACROSS_PARTITIONS) { - int maxPendingMsgsForOnePartition = maxPendingMessagesAcrossPartitions / numPartitions; - maxPendingMessages = (maxPendingMessages == DEFAULT_MAX_PENDING_MESSAGES) - ? maxPendingMsgsForOnePartition - : Math.min(maxPendingMessages, maxPendingMsgsForOnePartition); + if (conf.isMaxPendingMessagesAcrossPartitionsConfigured() && maxPendingMessagesAcrossPartitions > 0) { + // Never divide down to 0: a budget smaller than the partition count still asks for the + // smallest possible queue, not for the queue bound to be removed altogether. + int maxPendingMsgsForOnePartition = + Math.max(1, maxPendingMessagesAcrossPartitions / numPartitions); + // A per-producer limit the application set wins, including an explicit 0 ("no message-count + // limit"). Only a limit it left unset — or one filled in as a default — adopts the share. + maxPendingMessages = conf.isMaxPendingMessagesConfigured() + ? Math.min(maxPendingMessages, maxPendingMsgsForOnePartition) + : maxPendingMsgsForOnePartition; conf.setMaxPendingMessages(maxPendingMessages); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java index 6d6c2081e2dda..bc5abdf0d6868 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java @@ -56,14 +56,6 @@ public class ProducerBuilderImpl implements ProducerBuilder { private ProducerConfigurationData conf; private Schema schema; private List interceptorList; - /** - * Whether the application configured the pending-message limits. Their unset value is 0, which is - * also a meaningful explicit value ("no message-count limit"), so the configuration alone cannot - * tell the two apart. See - * {@link PulsarClientImpl#applyNoMemoryLimitProducerDefaults(ProducerConfigurationData, boolean, boolean)}. - */ - private boolean maxPendingMessagesConfigured; - private boolean maxPendingMessagesAcrossPartitionsConfigured; public ProducerBuilderImpl(PulsarClientImpl client, Schema schema) { this(client, new ProducerConfigurationData(), schema); @@ -86,10 +78,7 @@ public ProducerBuilder schema(Schema schema) { @Override public ProducerBuilder clone() { - ProducerBuilderImpl copy = new ProducerBuilderImpl<>(client, conf.clone(), schema); - copy.maxPendingMessagesConfigured = maxPendingMessagesConfigured; - copy.maxPendingMessagesAcrossPartitionsConfigured = maxPendingMessagesAcrossPartitionsConfigured; - return copy; + return new ProducerBuilderImpl<>(client, conf.clone(), schema); } @Override @@ -119,8 +108,7 @@ public CompletableFuture> createAsync() { return FutureUtil.failedFuture(pce); } - ProducerConfigurationData producerConf = client.applyNoMemoryLimitProducerDefaults(conf, - maxPendingMessagesConfigured, maxPendingMessagesAcrossPartitionsConfigured); + ProducerConfigurationData producerConf = client.applyNoMemoryLimitProducerDefaults(conf); return interceptorList == null || interceptorList.size() == 0 ? client.createProducerAsync(producerConf, schema, null) @@ -129,13 +117,19 @@ public CompletableFuture> createAsync() { @Override public ProducerBuilder loadConf(Map config) { + // A limit present in the map was configured by the application, even when its value is the + // same as the unset one. loadData builds a new configuration instance by replaying every + // property through its setters, so the markers have to be carried over rather than read off + // the result. + boolean maxPendingMessagesConfigured = + conf.isMaxPendingMessagesConfigured() || config.containsKey("maxPendingMessages"); + boolean maxPendingMessagesAcrossPartitionsConfigured = + conf.isMaxPendingMessagesAcrossPartitionsConfigured() + || config.containsKey("maxPendingMessagesAcrossPartitions"); conf = ConfigurationDataUtils.loadData( config, conf, ProducerConfigurationData.class); - // A limit present in the map was configured by the application, even when its value is the - // same as the unset one. loadData builds a new configuration instance, so this cannot be - // tracked in the configuration itself. - maxPendingMessagesConfigured |= config.containsKey("maxPendingMessages"); - maxPendingMessagesAcrossPartitionsConfigured |= config.containsKey("maxPendingMessagesAcrossPartitions"); + conf.setMaxPendingMessagesConfigured(maxPendingMessagesConfigured); + conf.setMaxPendingMessagesAcrossPartitionsConfigured(maxPendingMessagesAcrossPartitionsConfigured); return this; } @@ -161,7 +155,6 @@ public ProducerBuilder sendTimeout(int sendTimeout, @NonNull TimeUnit unit) { @Override public ProducerBuilder maxPendingMessages(int maxPendingMessages) { conf.setMaxPendingMessages(maxPendingMessages); - maxPendingMessagesConfigured = true; return this; } @@ -169,7 +162,6 @@ public ProducerBuilder maxPendingMessages(int maxPendingMessages) { @Override public ProducerBuilder maxPendingMessagesAcrossPartitions(int maxPendingMessagesAcrossPartitions) { conf.setMaxPendingMessagesAcrossPartitions(maxPendingMessagesAcrossPartitions); - maxPendingMessagesAcrossPartitionsConfigured = true; return this; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java index fd626c24d4715..e738307e0b00a 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java @@ -404,28 +404,29 @@ public CompletableFuture> createProducerAsync(ProducerConfigurat * *

These are defaults, not a floor. A limit the application configured is always kept — including * an explicit {@code 0}, which is how an application asks for no message-count limit at all. Only a - * limit that was never configured is filled in, which is why the caller passes in what it saw - * rather than letting this method infer it: {@code 0} is both the unset value and a meaningful - * explicit one. + * limit that was never configured is filled in, which is why this reads the markers the + * configuration carries rather than inferring it from the values: {@code 0} is both the unset value + * and a meaningful explicit one. * - *

Note that on a partitioned topic a filled-in across-partitions budget is still divided between - * the partitions afterwards, which can lower an explicitly configured per-producer limit. + *

What is filled in here stays marked as unconfigured, so on a partitioned topic + * {@link PartitionedProducerImpl} divides only a budget the application actually asked for. A + * filled-in budget never lowers a per-producer limit that was asked for. * - *

Called by {@link ProducerBuilderImpl}, which is what knows whether a limit was configured. + *

Called by {@link ProducerBuilderImpl}. * - * @param conf the requested producer configuration - * @param maxPendingMessagesConfigured whether the application configured {@code maxPendingMessages} - * @param maxPendingMessagesAcrossPartitionsConfigured whether the application configured - * {@code maxPendingMessagesAcrossPartitions} + * @param conf the requested producer configuration, carrying the markers that say which limits the + * application configured * @return the configuration to create the producer with; a resolved copy when a default applies, * otherwise {@code conf} unchanged */ - public ProducerConfigurationData applyNoMemoryLimitProducerDefaults(ProducerConfigurationData conf, - boolean maxPendingMessagesConfigured, boolean maxPendingMessagesAcrossPartitionsConfigured) { + public ProducerConfigurationData applyNoMemoryLimitProducerDefaults(ProducerConfigurationData conf) { // A limit that is already positive was configured by definition, whichever way the - // configuration was populated. The flags only tell an explicit 0 apart from an unset one. - maxPendingMessagesConfigured |= conf.getMaxPendingMessages() > 0; - maxPendingMessagesAcrossPartitionsConfigured |= conf.getMaxPendingMessagesAcrossPartitions() > 0; + // configuration was populated. The markers only tell an explicit 0 apart from an unset one. + boolean maxPendingMessagesConfigured = + conf.isMaxPendingMessagesConfigured() || conf.getMaxPendingMessages() > 0; + boolean maxPendingMessagesAcrossPartitionsConfigured = + conf.isMaxPendingMessagesAcrossPartitionsConfigured() + || conf.getMaxPendingMessagesAcrossPartitions() > 0; if ((maxPendingMessagesConfigured && maxPendingMessagesAcrossPartitionsConfigured) || memoryLimitController.isMemoryLimited()) { return conf; @@ -437,10 +438,8 @@ public ProducerConfigurationData applyNoMemoryLimitProducerDefaults(ProducerConf if (maxPendingMessagesAcrossPartitionsConfigured) { maxPendingMessagesAcrossPartitions = conf.getMaxPendingMessagesAcrossPartitions(); } else if (maxPendingMessages == 0) { - // The application configured no per-producer limit. Filling in a partitions budget would - // put one back, because a partitioned producer derives its per-partition limit from it, so - // a single maxPendingMessages(0) is enough to ask for a producer with no message-count - // limit whatever the topic's shape. + // The application asked for no per-producer limit at all, so there is no queue for a + // budget to bound. Leaving it unset keeps the resolved configuration honest about that. maxPendingMessagesAcrossPartitions = 0; } else { maxPendingMessagesAcrossPartitions = @@ -458,6 +457,12 @@ public ProducerConfigurationData applyNoMemoryLimitProducerDefaults(ProducerConf ProducerConfigurationData resolved = conf.clone(); resolved.setMaxPendingMessages(maxPendingMessages); resolved.setMaxPendingMessagesAcrossPartitions(maxPendingMessagesAcrossPartitions); + // The setters mark whatever they are given as configured, so restore the markers: what is + // filled in here is a default, and a partitioned producer still has to be able to tell it apart + // from a limit the application asked for, so that a budget it did ask for is the one that gets + // divided. + resolved.setMaxPendingMessagesConfigured(maxPendingMessagesConfigured); + resolved.setMaxPendingMessagesAcrossPartitionsConfigured(maxPendingMessagesAcrossPartitionsConfigured); if (log.isDebugEnabled()) { log.debug("[{}] Client memory limit is disabled, applying default producer pending message limits." + " maxPendingMessages: {}, maxPendingMessagesAcrossPartitions: {}", diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java index cec15904f71a6..1d085e5f1c047 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java @@ -109,6 +109,30 @@ public class ProducerConfigurationData implements Serializable, Cloneable { ) private int maxPendingMessagesAcrossPartitions = DEFAULT_MAX_PENDING_MESSAGES_ACROSS_PARTITIONS; + /** + * Whether the application configured {@link #maxPendingMessages}, and whether it configured + * {@link #maxPendingMessagesAcrossPartitions}. + * + *

The unset value of both limits is {@code 0}, which is also a meaningful explicit value ("no + * message-count limit" and "no across-partitions budget"), so the value alone cannot tell the two + * apart. Recording it here rather than on the builder lets everything that resolves these limits + * read the same answer, including {@code PartitionedProducerImpl}, which only sees the + * configuration. + * + *

Setting either limit through its setter marks it as configured, so a configuration populated + * directly rather than through {@code ProducerBuilderImpl} behaves the same way. + * + *

Deliberately not part of the serialized configuration: {@code loadConf} rebuilds the instance + * by replaying every property through its setters, which would mark both limits as configured + * whatever the application passed, so {@code ProducerBuilderImpl} restores them across that + * round-trip. {@code PulsarClientImpl} likewise restores them after filling in a default, which is + * not application input. + */ + @JsonIgnore + private boolean maxPendingMessagesConfigured; + @JsonIgnore + private boolean maxPendingMessagesAcrossPartitionsConfigured; + @ApiModelProperty( name = "messageRoutingMode", value = "Message routing logic for producers on [partitioned topics]" @@ -248,6 +272,7 @@ public void setProducerName(String producerName) { public void setMaxPendingMessages(int maxPendingMessages) { checkArgument(maxPendingMessages >= 0, "maxPendingMessages needs to be >= 0"); this.maxPendingMessages = maxPendingMessages; + this.maxPendingMessagesConfigured = true; } /** @@ -262,6 +287,7 @@ public void setMaxPendingMessagesAcrossPartitions(int maxPendingMessagesAcrossPa checkArgument(maxPendingMessagesAcrossPartitions >= 0, "maxPendingMessagesAcrossPartitions needs to be >= 0"); this.maxPendingMessagesAcrossPartitions = maxPendingMessagesAcrossPartitions; + this.maxPendingMessagesAcrossPartitionsConfigured = true; } public void setBatchingMaxMessages(int batchingMaxMessages) { diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PartitionedProducerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PartitionedProducerImplTest.java index ce2c200344d96..c7de9c7866a54 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PartitionedProducerImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PartitionedProducerImplTest.java @@ -300,14 +300,21 @@ public void testMaxPendingQueueSize() throws Exception { clientImpl, topicName, producerConfData, 1, null, null, null); assertEquals(partitionedProducerImpl.getConfiguration().getMaxPendingMessages(), 10); - // Test set MaxPendingMessagesAcrossPartitions=5 - producerConfData.setMaxPendingMessages(ProducerConfigurationData.DEFAULT_MAX_PENDING_MESSAGES); + // Test set MaxPendingMessagesAcrossPartitions=5 with maxPendingMessages left unset. A fresh + // configuration is required to express "unset": setting maxPendingMessages back to 0 would mean + // "no message-count limit", which is a limit of its own and would win over the budget's share. + producerConfData = new ProducerConfigurationData(); + producerConfData.setMessageRoutingMode(MessageRoutingMode.CustomPartition); + producerConfData.setCustomMessageRouter(new CustomMessageRouter()); producerConfData.setMaxPendingMessagesAcrossPartitions(5); partitionedProducerImpl = new PartitionedProducerImpl( clientImpl, topicName, producerConfData, 1, null, null, null); assertEquals(partitionedProducerImpl.getConfiguration().getMaxPendingMessages(), 5); // Test set maxPendingMessage=10 and MaxPendingMessagesAcrossPartitions=10 with 2 partitions + producerConfData = new ProducerConfigurationData(); + producerConfData.setMessageRoutingMode(MessageRoutingMode.CustomPartition); + producerConfData.setCustomMessageRouter(new CustomMessageRouter()); producerConfData.setMaxPendingMessages(10); producerConfData.setMaxPendingMessagesAcrossPartitions(10); partitionedProducerImpl = new PartitionedProducerImpl( diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java index 6033ebc1146e4..92d41a6932325 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java @@ -19,7 +19,6 @@ package org.apache.pulsar.client.impl; import static org.mockito.Mockito.any; -import static org.mockito.Mockito.anyBoolean; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -66,8 +65,8 @@ public void setup() { // The builder asks the client to fill in the pending-message defaults before creating the // producer; on a mock that would otherwise hand back a null configuration. - when(client.applyNoMemoryLimitProducerDefaults(any(ProducerConfigurationData.class), anyBoolean(), - anyBoolean())).thenAnswer(invocation -> invocation.getArgument(0)); + when(client.applyNoMemoryLimitProducerDefaults(any(ProducerConfigurationData.class))) + .thenAnswer(invocation -> invocation.getArgument(0)); when(client.createProducerAsync( any(ProducerConfigurationData.class), any(Schema.class), eq(null))) @@ -138,8 +137,8 @@ public void testLoadConfWithAPositiveMaxPendingMessages() { producerBuilderImpl.loadConf(Collections.singletonMap("maxPendingMessages", 5000)); assertEquals(producerBuilderImpl.getConf().getMaxPendingMessages(), 5000); - assertTrue(producerBuilderImpl.isMaxPendingMessagesConfigured()); - assertFalse(producerBuilderImpl.isMaxPendingMessagesAcrossPartitionsConfigured()); + assertTrue(producerBuilderImpl.getConf().isMaxPendingMessagesConfigured()); + assertFalse(producerBuilderImpl.getConf().isMaxPendingMessagesAcrossPartitionsConfigured()); } @Test diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/ContextImplTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/ContextImplTest.java index 071886698fb45..dbdb4eb1bc91f 100644 --- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/ContextImplTest.java +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/ContextImplTest.java @@ -20,7 +20,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.same; @@ -111,8 +110,8 @@ public void setup() throws PulsarClientException { invocation -> new ProducerBuilderImpl(client, invocation.getArgument(0))); // The builder asks the client to fill in the pending-message defaults before creating the // producer; on a mock that would otherwise hand back a null configuration. - when(client.applyNoMemoryLimitProducerDefaults(any(ProducerConfigurationData.class), anyBoolean(), - anyBoolean())).thenAnswer(invocation -> invocation.getArgument(0)); + when(client.applyNoMemoryLimitProducerDefaults(any(ProducerConfigurationData.class))) + .thenAnswer(invocation -> invocation.getArgument(0)); when(client.createProducerAsync(any(ProducerConfigurationData.class), any(), any())) .thenReturn(CompletableFuture.completedFuture(producer)); when(client.getSchema(anyString())).thenReturn(CompletableFuture.completedFuture(Optional.empty())); diff --git a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceProducerTest.java b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceProducerTest.java index 277f72bcafc03..6bd9ec532134e 100644 --- a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceProducerTest.java +++ b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceProducerTest.java @@ -260,8 +260,8 @@ public void testPendingMessageLimitsAreLeftToTheClientWhenUnset() throws Excepti Assert.assertNull(producer.maxOutstanding); Assert.assertNull(producer.maxPendingMessagesAcrossPartitions); // pulsar-perf must not configure either limit, so that they stay the client's to decide. - Assert.assertFalse(builder.isMaxPendingMessagesConfigured()); - Assert.assertFalse(builder.isMaxPendingMessagesAcrossPartitionsConfigured()); + Assert.assertFalse(builder.getConf().isMaxPendingMessagesConfigured()); + Assert.assertFalse(builder.getConf().isMaxPendingMessagesAcrossPartitionsConfigured()); // The client resolves its no-memory-limit defaults when the producer is created rather than on // the builder, so the effective configuration is where they have to show up. From 50132fdd9edff464fb570bb0d45f65e64921acee Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Mon, 24 Aug 2026 11:14:43 +0800 Subject: [PATCH 185/213] [improve][broker] Change partition metadata not-found log to warn (#26381) --- .../org/apache/pulsar/broker/service/ServerCnx.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index b23da296586e2..82e9c3f7f405e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -753,8 +753,15 @@ protected void handlePartitionMetadataRequest(CommandPartitionedTopicMetadata pa topicExistsInfo.recycle(); }).exceptionally(ex -> { lookupSemaphore.release(); - log.error("{} {} Failed to get partition metadata", topicName, - ServerCnx.this.toString(), ex); + Throwable actEx = FutureUtil.unwrapCompletionException(ex); + if (actEx instanceof WebApplicationException restException + && restException.getResponse().getStatus() == NOT_FOUND.getStatusCode()) { + log.warn("{} {} Failed to get partition metadata for nonexistent resource: {}", + topicName, ServerCnx.this, actEx.getMessage()); + } else { + log.error("{} {} Failed to get partition metadata", topicName, + ServerCnx.this, ex); + } writeAndFlush( Commands.newPartitionMetadataResponse(ServerError.MetadataError, "Failed to get partition metadata", From c7c559bca85d4a9d87b2dad594480c4c533d43d5 Mon Sep 17 00:00:00 2001 From: David Kjerrumgaard <35466513+david-streamlio@users.noreply.github.com> Date: Wed, 26 Aug 2026 11:21:26 +0800 Subject: [PATCH 186/213] [fix][fn] Honour negativeAckRedeliveryDelayMs in the Go function runtime (#26415) (cherry picked from commit 12b86b18171bec98ed686ddbc258664761ce8978) --- pulsar-function-go/pf/instance.go | 65 +++++++++++++++--------- pulsar-function-go/pf/nackDelay_test.go | 66 +++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 22 deletions(-) create mode 100644 pulsar-function-go/pf/nackDelay_test.go diff --git a/pulsar-function-go/pf/instance.go b/pulsar-function-go/pf/instance.go index 2cdfc8a6e9497..49e156a4177fd 100644 --- a/pulsar-function-go/pf/instance.go +++ b/pulsar-function-go/pf/instance.go @@ -303,6 +303,21 @@ func (gi *goInstance) getProducer(topicName string) (pulsar.Producer, error) { return producer, err } +// resolveNackRedeliveryDelay returns the negative-ack redelivery delay to apply, or zero to leave +// the client default in place. +// +// SourceSpec.NegativeAckRedeliveryDelayMs is a proto3 scalar with no presence, so an unset field +// reads as 0. Only a positive value is applied, matching the guard the Java runtime uses in +// JavaInstanceRunnable; a zero left in ConsumerOptions is treated by the client as unset, so the +// default applies either way. +func resolveNackRedeliveryDelay(delayMs uint64) time.Duration { + if delayMs == 0 { + return 0 + } + + return time.Duration(delayMs) * time.Millisecond +} + func (gi *goInstance) setupConsumer() (chan pulsar.ConsumerMessage, error) { subscriptionType := pulsar.Shared if int32(gi.context.instanceConf.funcDetails.Source.SubscriptionType) == pb.SubscriptionType_value["FAILOVER"] { @@ -320,6 +335,8 @@ func (gi *goInstance) setupConsumer() (chan pulsar.ConsumerMessage, error) { funcDetails.Namespace, funcDetails.Name), gi.context.instanceConf.instanceID) + nackRedeliveryDelay := resolveNackRedeliveryDelay(funcDetails.Source.NegativeAckRedeliveryDelayMs) + channel := make(chan pulsar.ConsumerMessage) var ( @@ -338,39 +355,43 @@ func (gi *goInstance) setupConsumer() (chan pulsar.ConsumerMessage, error) { if consumerConf.ReceiverQueueSize != nil { if consumerConf.IsRegexPattern { consumer, err = gi.client.Subscribe(pulsar.ConsumerOptions{ - TopicsPattern: topicName.Name, - ReceiverQueueSize: int(consumerConf.ReceiverQueueSize.Value), - SubscriptionName: subscriptionName, - Properties: properties, - Type: subscriptionType, - MessageChannel: channel, + TopicsPattern: topicName.Name, + ReceiverQueueSize: int(consumerConf.ReceiverQueueSize.Value), + SubscriptionName: subscriptionName, + Properties: properties, + Type: subscriptionType, + MessageChannel: channel, + NackRedeliveryDelay: nackRedeliveryDelay, }) } else { consumer, err = gi.client.Subscribe(pulsar.ConsumerOptions{ - Topic: topicName.Name, - SubscriptionName: subscriptionName, - Properties: properties, - Type: subscriptionType, - ReceiverQueueSize: int(consumerConf.ReceiverQueueSize.Value), - MessageChannel: channel, + Topic: topicName.Name, + SubscriptionName: subscriptionName, + Properties: properties, + Type: subscriptionType, + ReceiverQueueSize: int(consumerConf.ReceiverQueueSize.Value), + MessageChannel: channel, + NackRedeliveryDelay: nackRedeliveryDelay, }) } } else { if consumerConf.IsRegexPattern { consumer, err = gi.client.Subscribe(pulsar.ConsumerOptions{ - TopicsPattern: topicName.Name, - SubscriptionName: subscriptionName, - Properties: properties, - Type: subscriptionType, - MessageChannel: channel, + TopicsPattern: topicName.Name, + SubscriptionName: subscriptionName, + Properties: properties, + Type: subscriptionType, + MessageChannel: channel, + NackRedeliveryDelay: nackRedeliveryDelay, }) } else { consumer, err = gi.client.Subscribe(pulsar.ConsumerOptions{ - Topic: topicName.Name, - SubscriptionName: subscriptionName, - Properties: properties, - Type: subscriptionType, - MessageChannel: channel, + Topic: topicName.Name, + SubscriptionName: subscriptionName, + Properties: properties, + Type: subscriptionType, + MessageChannel: channel, + NackRedeliveryDelay: nackRedeliveryDelay, }) } diff --git a/pulsar-function-go/pf/nackDelay_test.go b/pulsar-function-go/pf/nackDelay_test.go new file mode 100644 index 0000000000000..0ec704eca7e4c --- /dev/null +++ b/pulsar-function-go/pf/nackDelay_test.go @@ -0,0 +1,66 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package pf + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +// The Go runtime nacks on failure but never configured the delay, so the client default of 60s +// applied regardless of SourceSpec.NegativeAckRedeliveryDelayMs. +func TestResolveNackRedeliveryDelay(t *testing.T) { + tests := []struct { + name string + delayMs uint64 + expected time.Duration + }{ + { + // proto3 scalar with no presence: unset reads as 0. Returning zero leaves + // ConsumerOptions at its zero value, which the client treats as unset. + name: "unset leaves the client default", + delayMs: 0, + expected: 0, + }, + { + name: "milliseconds are converted to a duration", + delayMs: 5000, + expected: 5 * time.Second, + }, + { + name: "sub-second values are preserved", + delayMs: 250, + expected: 250 * time.Millisecond, + }, + { + name: "the client default expressed explicitly still round-trips", + delayMs: 60000, + expected: time.Minute, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.expected, resolveNackRedeliveryDelay(test.delayMs)) + }) + } +} From a86cf32ad1abf5a91d04a865f4fec5bd3437274e Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Wed, 26 Aug 2026 11:27:17 +0800 Subject: [PATCH 187/213] [fix][broker] Do not log an error when the tenant does not exist (#26361) (cherry picked from commit a25ae29c2afbd62d6a4b89ad359df35ae0404de0) --- .../MultiRolesTokenAuthorizationProvider.java | 27 ++-- .../PulsarAuthorizationProvider.java | 25 ++-- .../broker/authorization/LogCapture.java | 94 +++++++++++++ ...tiRolesTokenAuthorizationProviderTest.java | 54 ++++++++ .../PulsarAuthorizationProviderTest.java | 125 ++++++++++++++++++ 5 files changed, 306 insertions(+), 19 deletions(-) create mode 100644 pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/LogCapture.java create mode 100644 pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProviderTest.java diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java index 6b46289af4811..272f6ee868dcf 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java @@ -28,6 +28,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.function.Function; @@ -120,6 +121,18 @@ public CompletableFuture validateTenantAdminAccess(String tenantName, S return pulsarResources.getTenantResources() .getTenantAsync(tenantName) + // Failing to read the tenant is a broker-side fault, so it is handled here, on the + // stage that can actually fail. Keeping this handler off the stage below prevents the + // expected "tenant does not exist" rejection from being reported as an error. + .exceptionally(ex -> { + Throwable cause = FutureUtil.unwrapCompletionException(ex); + if (cause instanceof MetadataStoreException.NotFoundException) { + log.warn("Failed to get tenant info data for non existing tenant {}", tenantName); + return Optional.empty(); + } + log.error("Failed to get tenant {}", tenantName, cause); + throw new RestException(cause); + }) .thenCompose(op -> { if (op.isPresent()) { TenantInfo tenantInfo = op.get(); @@ -129,17 +142,11 @@ public CompletableFuture validateTenantAdminAccess(String tenantName, S return CompletableFuture.completedFuture(roles.stream() .anyMatch(n -> tenantInfo.getAdminRoles().contains(n))); - } else { - throw new RestException(Response.Status.NOT_FOUND, "Tenant does not exist"); } - }).exceptionally(ex -> { - Throwable cause = ex.getCause(); - if (cause instanceof MetadataStoreException.NotFoundException) { - log.warn("Failed to get tenant info data for non existing tenant {}", tenantName); - throw new RestException(Response.Status.NOT_FOUND, "Tenant does not exist"); - } - log.error("Failed to get tenant {}", tenantName, cause); - throw new RestException(cause); + // A client naming a tenant that does not exist is a client error, not a broker + // fault: reject it without logging. Any client can trigger this at will, and the + // caller (e.g. ServerCnx) already logs the rejection at its own level. + throw new RestException(Response.Status.NOT_FOUND, "Tenant does not exist"); }); }); } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java index e4f2ac7e8cd16..8837b340f9b79 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProvider.java @@ -26,6 +26,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.stream.Stream; @@ -716,20 +717,26 @@ public CompletableFuture validateTenantAdminAccess(String tenantName, S } return pulsarResources.getTenantResources() .getTenantAsync(tenantName) - .thenCompose(op -> { - if (op.isPresent()) { - return isTenantAdmin(tenantName, role, op.get(), authData); - } else { - throw new RestException(Response.Status.NOT_FOUND, "Tenant does not exist"); - } - }).exceptionally(ex -> { - Throwable cause = ex.getCause(); + // Failing to read the tenant is a broker-side fault, so it is handled here, on the + // stage that can actually fail. Keeping this handler off the stage below prevents the + // expected "tenant does not exist" rejection from being reported as an error. + .exceptionally(ex -> { + Throwable cause = FutureUtil.unwrapCompletionException(ex); if (cause instanceof NotFoundException) { log.warn("Failed to get tenant info data for non existing tenant {}", tenantName); - throw new RestException(Response.Status.NOT_FOUND, "Tenant does not exist"); + return Optional.empty(); } log.error("Failed to get tenant {}", tenantName, cause); throw new RestException(cause); + }) + .thenCompose(op -> { + if (op.isPresent()) { + return isTenantAdmin(tenantName, role, op.get(), authData); + } + // A client naming a tenant that does not exist is a client error, not a broker + // fault: reject it without logging. Any client can trigger this at will, and the + // caller (e.g. ServerCnx) already logs the rejection at its own level. + throw new RestException(Response.Status.NOT_FOUND, "Tenant does not exist"); }); }); } diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/LogCapture.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/LogCapture.java new file mode 100644 index 0000000000000..fb715d268f46b --- /dev/null +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/LogCapture.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.authorization; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.appender.AbstractAppender; +import org.apache.logging.log4j.core.config.LoggerConfig; +import org.apache.logging.log4j.core.layout.PatternLayout; + +/** + * Captures the Log4j2 events emitted by a single logger, so that tests can assert on the level a + * given condition is reported at. slog resolves to its Log4j2 backend whenever log4j-core is on the + * classpath, so this captures slog output too. + */ +final class LogCapture extends AbstractAppender implements AutoCloseable { + + private static final AtomicInteger ID = new AtomicInteger(); + + private final List events = Collections.synchronizedList(new ArrayList<>()); + private final String loggerName; + private final LoggerConfig loggerConfig; + private final LoggerContext context; + + /** + * Starts capturing the events logged by {@code loggerOwner}'s logger. The capture is released on + * {@link #close()}. + */ + static LogCapture attach(Class loggerOwner) { + LoggerContext context = (LoggerContext) LogManager.getContext(false); + LoggerConfig loggerConfig = context.getConfiguration().getLoggerConfig(loggerOwner.getName()); + LogCapture capture = new LogCapture(loggerOwner.getName(), loggerConfig, context); + capture.start(); + // The resolved LoggerConfig is often an ancestor (usually root), so this appender may also see + // events from unrelated loggers. append() filters them out by name. + loggerConfig.addAppender(capture, Level.ALL, null); + context.updateLoggers(); + return capture; + } + + private LogCapture(String loggerName, LoggerConfig loggerConfig, LoggerContext context) { + super("LogCapture-" + ID.incrementAndGet(), null, PatternLayout.createDefaultLayout(), false, null); + this.loggerName = loggerName; + this.loggerConfig = loggerConfig; + this.context = context; + } + + @Override + public void append(LogEvent event) { + if (loggerName.equals(event.getLoggerName())) { + events.add(event.toImmutable()); + } + } + + /** Returns the formatted messages captured at {@code level}, in the order they were logged. */ + List messagesAt(Level level) { + synchronized (events) { + return events.stream() + .filter(event -> event.getLevel() == level) + .map(event -> event.getMessage().getFormattedMessage()) + .collect(Collectors.toList()); + } + } + + @Override + public void close() { + stop(); + loggerConfig.removeAppender(getName()); + context.updateLoggers(); + } +} diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProviderTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProviderTest.java index 094152a5c44a2..a2db9669c35db 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProviderTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProviderTest.java @@ -19,24 +19,32 @@ package org.apache.pulsar.broker.authorization; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import static org.testng.Assert.expectThrows; import io.jsonwebtoken.Jwts; import io.jsonwebtoken.SignatureAlgorithm; +import java.util.List; +import java.util.Optional; import java.util.Properties; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.function.Function; +import javax.ws.rs.core.Response; import javax.crypto.SecretKey; import lombok.Cleanup; +import org.apache.logging.log4j.Level; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription; import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; import org.apache.pulsar.broker.resources.PulsarResources; +import org.apache.pulsar.broker.resources.TenantResources; +import org.apache.pulsar.common.util.RestException; import org.testng.annotations.Test; public class MultiRolesTokenAuthorizationProviderTest { @@ -411,4 +419,50 @@ public String getHttpHeader(String name) { assertTrue(ex.getCause().getMessage().contains( "The subscription name needs to be prefixed by the authentication role")); } + + /** + * A client asking about a tenant that does not exist is a client-side 404, not a broker fault, so + * it must not be reported at ERROR. Any client can trigger it at will simply by mistyping a tenant. + */ + @Test + public void testMissingTenantIsNotLoggedAtError() throws Exception { + String tenant = "non-existent-tenant"; + TenantResources tenantResources = mock(TenantResources.class); + when(tenantResources.getTenantAsync(tenant)) + .thenReturn(CompletableFuture.completedFuture(Optional.empty())); + PulsarResources pulsarResources = mock(PulsarResources.class); + when(pulsarResources.getTenantResources()).thenReturn(tenantResources); + + SecretKey secretKey = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256); + String token = Jwts.builder().claim("sub", new String[]{"user-a"}).signWith(secretKey).compact(); + + MultiRolesTokenAuthorizationProvider provider = new MultiRolesTokenAuthorizationProvider(); + provider.initialize(new ServiceConfiguration(), pulsarResources); + + AuthenticationDataSource ads = new AuthenticationDataSource() { + @Override + public boolean hasDataFromHttp() { + return true; + } + + @Override + public String getHttpHeader(String name) { + if (name.equals("Authorization")) { + return "Bearer " + token; + } else { + throw new IllegalArgumentException("Wrong HTTP header"); + } + } + }; + + try (LogCapture logs = LogCapture.attach(MultiRolesTokenAuthorizationProvider.class)) { + CompletableFuture future = provider.validateTenantAdminAccess(tenant, "user-a", ads); + + ExecutionException ee = expectThrows(ExecutionException.class, future::get); + assertEquals(((RestException) ee.getCause()).getResponse().getStatus(), + Response.Status.NOT_FOUND.getStatusCode()); + assertEquals(logs.messagesAt(Level.ERROR), List.of(), + "A tenant that does not exist must not be logged at ERROR"); + } + } } diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProviderTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProviderTest.java new file mode 100644 index 0000000000000..540c1ca252626 --- /dev/null +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/PulsarAuthorizationProviderTest.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.authorization; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.expectThrows; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import javax.ws.rs.core.Response; +import org.apache.logging.log4j.Level; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.resources.PulsarResources; +import org.apache.pulsar.broker.resources.TenantResources; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.TenantInfo; +import org.apache.pulsar.common.policies.data.TopicOperation; +import org.apache.pulsar.common.util.RestException; +import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.testng.annotations.Test; + +public class PulsarAuthorizationProviderTest { + + private static final String MISSING_TENANT = "non-existent-tenant"; + private static final String ROLE = "some-role"; + + private static PulsarAuthorizationProvider providerReturning( + CompletableFuture> tenantLookup) throws Exception { + TenantResources tenantResources = mock(TenantResources.class); + when(tenantResources.getTenantAsync(MISSING_TENANT)).thenReturn(tenantLookup); + PulsarResources pulsarResources = mock(PulsarResources.class); + when(pulsarResources.getTenantResources()).thenReturn(tenantResources); + + PulsarAuthorizationProvider provider = new PulsarAuthorizationProvider(); + provider.initialize(new ServiceConfiguration(), pulsarResources); + return provider; + } + + /** A tenant that was never created: {@code getTenantAsync} succeeds with an empty Optional. */ + private static PulsarAuthorizationProvider providerWithMissingTenant() throws Exception { + return providerReturning(CompletableFuture.completedFuture(Optional.empty())); + } + + private static void assertNotFound(ExecutionException ee) { + RestException restException = (RestException) ee.getCause(); + assertEquals(restException.getResponse().getStatus(), Response.Status.NOT_FOUND.getStatusCode()); + } + + /** + * A client asking about a tenant that does not exist is a client-side 404, not a broker fault, so + * it must not be reported at ERROR. Any client can trigger it at will simply by mistyping a tenant. + */ + @Test + public void testMissingTenantIsNotLoggedAtErrorByValidateTenantAdminAccess() throws Exception { + PulsarAuthorizationProvider provider = providerWithMissingTenant(); + + try (LogCapture logs = LogCapture.attach(PulsarAuthorizationProvider.class)) { + CompletableFuture future = + provider.validateTenantAdminAccess(MISSING_TENANT, ROLE, null); + + assertNotFound(expectThrows(ExecutionException.class, future::get)); + assertEquals(logs.messagesAt(Level.ERROR), List.of(), + "A tenant that does not exist must not be logged at ERROR"); + } + } + + /** + * The path an actual LOOKUP takes: ServerCnx -> AuthorizationService -> allowTopicOperationAsync. + * One ERROR line per lookup here is what floods a broker when a misconfigured client hammers a + * cluster that does not host its tenant. + */ + @Test + public void testMissingTenantIsNotLoggedAtErrorByLookupAuthorization() throws Exception { + PulsarAuthorizationProvider provider = providerWithMissingTenant(); + TopicName topicName = TopicName.get("persistent://" + MISSING_TENANT + "/ns/topic"); + + try (LogCapture logs = LogCapture.attach(PulsarAuthorizationProvider.class)) { + CompletableFuture future = + provider.allowTopicOperationAsync(topicName, ROLE, TopicOperation.LOOKUP, null); + + assertNotFound(expectThrows(ExecutionException.class, future::get)); + assertEquals(logs.messagesAt(Level.ERROR), List.of(), + "A tenant that does not exist must not be logged at ERROR"); + } + } + + /** + * Guards the fix from over-reaching: a genuine metadata store failure is a broker-side fault and + * must still be reported at ERROR. + */ + @Test + public void testMetadataStoreFailureIsStillLoggedAtError() throws Exception { + PulsarAuthorizationProvider provider = providerReturning(CompletableFuture.failedFuture( + new MetadataStoreException("simulated metadata store failure"))); + + try (LogCapture logs = LogCapture.attach(PulsarAuthorizationProvider.class)) { + CompletableFuture future = + provider.validateTenantAdminAccess(MISSING_TENANT, ROLE, null); + + expectThrows(ExecutionException.class, future::get); + assertFalse(logs.messagesAt(Level.ERROR).isEmpty(), + "A metadata store failure must still be logged at ERROR"); + } + } +} From 52c901a34688b4d1dd36522c70bdfca7a73941c2 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Wed, 26 Aug 2026 18:10:35 +0800 Subject: [PATCH 188/213] [fix][test] Fix import order in authorization test --- .../authorization/MultiRolesTokenAuthorizationProviderTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProviderTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProviderTest.java index a2db9669c35db..9dbd75a6b026c 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProviderTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProviderTest.java @@ -33,8 +33,8 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.function.Function; -import javax.ws.rs.core.Response; import javax.crypto.SecretKey; +import javax.ws.rs.core.Response; import lombok.Cleanup; import org.apache.logging.log4j.Level; import org.apache.pulsar.broker.PulsarServerException; From 4f5d2a034d3bf35f0168216a1ad5619ad1205029 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Wed, 26 Aug 2026 18:25:38 +0800 Subject: [PATCH 189/213] [improve][broker] Add pulsar_subscription_storage_backlog_age_seconds metric (#26313) --- conf/broker.conf | 6 + conf/standalone.conf | 6 + .../pulsar/broker/ServiceConfiguration.java | 9 + .../pulsar/broker/service/BrokerService.java | 32 +++ .../persistent/PersistentSubscription.java | 2 + .../service/persistent/PersistentTopic.java | 255 +++++++++++++++++- .../AggregatedSubscriptionStats.java | 2 + .../prometheus/NamespaceStatsAggregator.java | 5 +- .../broker/stats/prometheus/TopicStats.java | 7 +- .../service/BacklogQuotaManagerTest.java | 183 ++++++++++++- .../NamespaceStatsAggregatorTest.java | 4 + .../policies/data/SubscriptionStats.java | 12 + .../data/stats/SubscriptionStatsImpl.java | 13 + .../data/stats/SubscriptionStatsImplTest.java | 19 +- 14 files changed, 540 insertions(+), 15 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index 72a5f50876fa0..7ee46dcd7a399 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -1812,6 +1812,12 @@ metricsServletTimeoutMs=30000 # Enable or disable broker bundles metrics. The default value is false. exposeBundlesMetricsInPrometheus=false +# Enable computing the age of the oldest unacknowledged message for each subscription and exposing it +# through topic stats and Prometheus. +# When disabled, the broker skips computing per-subscription backlog age and the admin API field +# SubscriptionStats.oldestBacklogMessageAgeSeconds remains -1. Default is false. +exposeSubscriptionBacklogAgeInPrometheus=false + ### --- Functions --- ### # Enable Functions Worker Service in Broker diff --git a/conf/standalone.conf b/conf/standalone.conf index 59fcbdf2aabc9..06cd3e38659af 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -1123,6 +1123,12 @@ exposePublisherStats=true # Default is false. exposePreciseBacklogInPrometheus=false +# Enable computing the age of the oldest unacknowledged message for each subscription and exposing it +# through topic stats and Prometheus. +# When disabled, the broker skips computing per-subscription backlog age and the admin API field +# SubscriptionStats.oldestBacklogMessageAgeSeconds remains -1. Default is false. +exposeSubscriptionBacklogAgeInPrometheus=false + # Enable splitting topic and partition label in Prometheus. # If enabled, a topic name will split into 2 parts, one is topic name without partition index, # another one is partition index, e.g. (topic=xxx, partition=0). diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index ba2a643ddadc9..d3f308ba7bfce 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -3520,6 +3520,15 @@ public double getLoadBalancerBandwidthOutResourceWeight() { ) private boolean exposeSubscriptionBacklogSizeInPrometheus = false; + @FieldContext( + category = CATEGORY_METRICS, + doc = "Enable computing the age of the oldest unacknowledged message for each subscription and exposing " + + "it through topic stats and Prometheus.\n" + + " When disabled, the broker skips computing per-subscription backlog age and " + + "SubscriptionStats.oldestBacklogMessageAgeSeconds remains -1. Default is false." + ) + private boolean exposeSubscriptionBacklogAgeInPrometheus = false; + @FieldContext( category = CATEGORY_METRICS, doc = "Enable splitting topic and partition label in Prometheus.\n" diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index da4a812e6cc97..d90b0c48674e6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -26,6 +26,7 @@ import static org.apache.pulsar.client.util.RetryMessageUtil.DLQ_GROUP_TOPIC_SUFFIX; import static org.apache.pulsar.client.util.RetryMessageUtil.RETRY_GROUP_TOPIC_SUFFIX; import static org.apache.pulsar.common.naming.SystemTopicNames.isTransactionInternalName; +import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Queues; import com.google.common.util.concurrent.RateLimiter; @@ -263,6 +264,7 @@ public class BrokerService implements Closeable { @Getter private final SingleThreadNonConcurrentFixedRateScheduler backlogQuotaChecker; + private final SingleThreadNonConcurrentFixedRateScheduler subscriptionBacklogAgeChecker; protected final AtomicReference lookupRequestSemaphore; @Getter @@ -394,6 +396,8 @@ public BrokerService(PulsarService pulsar, EventLoopGroup eventLoopGroup) throws new SingleThreadNonConcurrentFixedRateScheduler("pulsar-consumed-ledgers-monitor"); this.backlogQuotaManager = new BacklogQuotaManager(pulsar); this.backlogQuotaChecker = new SingleThreadNonConcurrentFixedRateScheduler("pulsar-backlog-quota-checker"); + this.subscriptionBacklogAgeChecker = + new SingleThreadNonConcurrentFixedRateScheduler("pulsar-subscription-backlog-age-checker"); this.authenticationService = new AuthenticationService(pulsar.getConfiguration(), pulsar.getOpenTelemetry().getOpenTelemetry()); this.topicFactory = createPersistentTopicFactory(); @@ -656,6 +660,7 @@ public void start() throws Exception { this.startCompactionMonitor(); this.startConsumedLedgersMonitor(); this.startBacklogQuotaChecker(); + this.startSubscriptionBacklogAgeChecker(); this.updateBrokerPublisherThrottlingMaxRate(); this.updateBrokerDispatchThrottlingMaxRate(); this.startCheckReplicationPolicies(); @@ -792,6 +797,19 @@ protected void startBacklogQuotaChecker() { } + protected void startSubscriptionBacklogAgeChecker() { + if (pulsar().getConfiguration().isExposeSubscriptionBacklogAgeInPrometheus()) { + final int interval = pulsar().getConfiguration().getBacklogQuotaCheckIntervalInSeconds(); + log.info("Scheduling a thread to refresh subscription backlog age in background after [{}] seconds", + interval); + subscriptionBacklogAgeChecker.scheduleAtFixedRateNonConcurrently( + catchingAndLoggingThrowables(() -> refreshSubscriptionBacklogAge().join()), interval, interval, + TimeUnit.SECONDS); + } else { + log.info("Subscription backlog age monitoring is disabled"); + } + } + public void close() throws IOException { try { closeAsync().get(); @@ -939,6 +957,7 @@ public CompletableFuture closeAsync() { compactionMonitor, consumedLedgersMonitor, backlogQuotaChecker, + subscriptionBacklogAgeChecker, topicOrderedExecutor, deduplicationSnapshotMonitor) .handle()); @@ -2505,6 +2524,19 @@ public void monitorBacklogQuota() { }); } + public CompletableFuture refreshSubscriptionBacklogAge() { + if (!pulsar.getConfiguration().isExposeSubscriptionBacklogAgeInPrometheus()) { + return CompletableFuture.completedFuture(null); + } + + List> futures = new ArrayList<>(); + forEachPersistentTopic(topic -> futures.add(topic.updateSubscriptionOldPositionInfo())); + return FutureUtil.waitForAll(futures).exceptionally(throwable -> { + log.warn("Error when refreshSubscriptionBacklogAge()", throwable); + return null; + }); + } + public CompletableFuture isTopicNsOwnedByBrokerAsync(TopicName topicName) { return pulsar.getNamespaceService().isServiceUnitOwnedAsync(topicName) .handle((hasOwnership, t) -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index 081c2cdf49bfc..63e7317062e69 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -1447,6 +1447,8 @@ public CompletableFuture getStatsAsync(GetStatsOptions ge } } subStats.msgBacklog = getNumberOfEntriesInBacklog(getStatsOptions.isGetPreciseBacklog()); + subStats.oldestBacklogMessageAgeSeconds = + topic.getBestEffortOldestUnacknowledgedMessageAgeSeconds(subName); if (getStatsOptions.isSubscriptionBacklogSize()) { subStats.backlogSize = topic.getManagedLedger() .getEstimatedBacklogSize(cursor.getMarkDeletedPosition()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index aebec1d0d31bb..e6e6866b50659 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -319,6 +319,8 @@ protected TopicStatsHelper initialValue() { PersistentTopic.class, OldestPositionInfo.class, "oldestPositionInfo"); + private volatile Map subscriptionOldestPositionInfos; + @Value private static class OldestPositionInfo { Position oldestCursorMarkDeletePosition; @@ -3864,6 +3866,44 @@ public long getBestEffortOldestUnacknowledgedMessageAgeSeconds() { } } + long getBestEffortOldestUnacknowledgedMessageAgeSeconds(String subscriptionName) { + if (!brokerService.pulsar().getConfiguration().isExposeSubscriptionBacklogAgeInPrometheus()) { + return -1; + } + Map positionInfos = subscriptionOldestPositionInfos; + if (positionInfos == null) { + return -1; + } + OldestPositionInfo positionInfo = positionInfos.get(subscriptionName); + if (positionInfo == null) { + return -1; + } else { + return TimeUnit.MILLISECONDS.toSeconds( + Clock.systemUTC().millis() - positionInfo.getPositionPublishTimestampInMillis()); + } + } + + private Map getSubscriptionOldestPositionInfos() { + Map positionInfos = subscriptionOldestPositionInfos; + if (positionInfos == null) { + synchronized (this) { + positionInfos = subscriptionOldestPositionInfos; + if (positionInfos == null) { + positionInfos = new ConcurrentHashMap<>(); + subscriptionOldestPositionInfos = positionInfos; + } + } + } + return positionInfos; + } + + private void clearSubscriptionOldestPositionInfos() { + Map positionInfos = subscriptionOldestPositionInfos; + if (positionInfos != null) { + positionInfos.clear(); + } + } + private void updateResultIfNewer(OldestPositionInfo updatedResult) { TIME_BASED_BACKLOG_QUOTA_CHECK_RESULT_UPDATER.updateAndGet(this, existingResult -> { @@ -3878,6 +3918,213 @@ private void updateResultIfNewer(OldestPositionInfo updatedResult) { } + private void updateSubscriptionResultIfNewer(String subscriptionName, OldestPositionInfo updatedResult) { + getSubscriptionOldestPositionInfos().compute(subscriptionName, + (__, existingResult) -> { + if (existingResult == null + || ManagedCursorContainer.DataVersion.compareVersions( + updatedResult.getDataVersion(), existingResult.getDataVersion()) > 0) { + return updatedResult; + } else { + return existingResult; + } + }); + } + + private void clearSubscriptionResultIfNewer(String subscriptionName, long dataVersion) { + Map positionInfos = subscriptionOldestPositionInfos; + if (positionInfos == null) { + return; + } + positionInfos.computeIfPresent(subscriptionName, + (__, existingResult) -> + ManagedCursorContainer.DataVersion.compareVersions( + dataVersion, existingResult.getDataVersion()) >= 0 ? null : existingResult); + } + + private boolean isSubscriptionOldPositionInfoReusable(String subscriptionName, Position markDeletePosition) { + Map positionInfos = subscriptionOldestPositionInfos; + OldestPositionInfo positionInfo = positionInfos == null ? null : positionInfos.get(subscriptionName); + return positionInfo != null + && markDeletePosition.compareTo(positionInfo.getOldestCursorMarkDeletePosition()) == 0 + && markDeletePosition.compareTo(ledger.getFirstPosition()) >= 0; + } + + private CompletableFuture updateSubscriptionOldPositionInfos(ManagedCursorContainer managedCursorContainer, + boolean preciseTimeBasedBacklogQuotaCheck) { + Set activeSubscriptionNames = ConcurrentHashMap.newKeySet(); + Set failedSubscriptionNames = ConcurrentHashMap.newKeySet(); + List> futures = new ArrayList<>(); + CursorInfo oldestCursorInfo = managedCursorContainer.getCursorWithOldestPosition(); + long dataVersion = oldestCursorInfo == null ? 0 : oldestCursorInfo.getVersion(); + + for (Map.Entry subscriptionEntry : subscriptions.entrySet()) { + String subscriptionName = subscriptionEntry.getKey(); + PersistentSubscription subscription = subscriptionEntry.getValue(); + ManagedCursor cursor = subscription.getCursor(); + if (!cursor.isDurable()) { + clearSubscriptionResultIfNewer(subscriptionName, dataVersion); + continue; + } + activeSubscriptionNames.add(subscriptionName); + Position markDeletePosition = cursor.getMarkDeletedPosition(); + if (markDeletePosition == null) { + clearSubscriptionResultIfNewer(subscriptionName, dataVersion); + continue; + } + if (isSubscriptionOldPositionInfoReusable(subscriptionName, markDeletePosition)) { + continue; + } + if (subscription.getNumberOfEntriesInBacklog(preciseTimeBasedBacklogQuotaCheck) <= 0) { + clearSubscriptionResultIfNewer(subscriptionName, dataVersion); + continue; + } + + if (preciseTimeBasedBacklogQuotaCheck) { + futures.add(updateSubscriptionOldPositionInfoPrecise( + subscriptionName, subscription, cursor, markDeletePosition, dataVersion, + failedSubscriptionNames, preciseTimeBasedBacklogQuotaCheck)); + } else { + try { + EstimateTimeBasedBacklogQuotaCheckResult checkResult = + estimatedTimeBasedBacklogQuotaCheck(markDeletePosition); + if (checkResult.getEstimatedOldestUnacknowledgedMessageTimestamp() != null) { + updateSubscriptionResultIfNewer(subscriptionName, + new OldestPositionInfo( + markDeletePosition, + cursor.getName(), + checkResult.getEstimatedOldestUnacknowledgedMessageTimestamp(), + dataVersion)); + } else { + clearSubscriptionResultIfNewer(subscriptionName, dataVersion); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + failedSubscriptionNames.add(subscriptionName); + clearSubscriptionResultIfNewer(subscriptionName, dataVersion); + break; + } catch (ExecutionException e) { + failedSubscriptionNames.add(subscriptionName); + clearSubscriptionResultIfNewer(subscriptionName, dataVersion); + } + } + } + + Map positionInfos = subscriptionOldestPositionInfos; + if (positionInfos != null) { + positionInfos.keySet().removeIf(subscriptionName -> !activeSubscriptionNames.contains(subscriptionName)); + } + if (futures.isEmpty()) { + logSubscriptionOldPositionInfoUpdateFailures(failedSubscriptionNames); + return CompletableFuture.completedFuture(null); + } + return FutureUtil.waitForAll(futures) + .thenRun(() -> logSubscriptionOldPositionInfoUpdateFailures(failedSubscriptionNames)); + } + + private void logSubscriptionOldPositionInfoUpdateFailures(Set failedSubscriptionNames) { + if (!failedSubscriptionNames.isEmpty()) { + log.warn("Failed to update subscription old position info for {} subscriptions", + failedSubscriptionNames.size()); + } + } + + private CompletableFuture updateSubscriptionOldPositionInfoPrecise(String subscriptionName, + PersistentSubscription subscription, + ManagedCursor cursor, + Position markDeletePosition, + long dataVersion, + Set failedSubscriptionNames, + boolean preciseBacklogQuotaCheck) { + Position position = ledger.getNextValidPosition(markDeletePosition); + if (position.compareTo(ledger.getLastConfirmedEntry()) > 0) { + clearSubscriptionResultIfNewer(subscriptionName, dataVersion); + return CompletableFuture.completedFuture(null); + } + + CompletableFuture future = new CompletableFuture<>(); + ledger.asyncReadEntry(position, new AsyncCallbacks.ReadEntryCallback() { + @Override + public void readEntryComplete(Entry entry, Object ctx) { + try { + if (isSubscriptionOldPositionInfoReadResultStillValid( + subscriptionName, subscription, cursor, markDeletePosition, dataVersion, + preciseBacklogQuotaCheck)) { + long entryTimestamp = Commands.getEntryTimestamp(entry.getDataBuffer()); + updateSubscriptionResultIfNewer(subscriptionName, + new OldestPositionInfo( + markDeletePosition, + cursor.getName(), + entryTimestamp, + dataVersion)); + } + future.complete(null); + } catch (Exception e) { + failedSubscriptionNames.add(subscriptionName); + clearSubscriptionResultIfNewer(subscriptionName, dataVersion); + future.complete(null); + } finally { + entry.release(); + } + } + + @Override + public void readEntryFailed(ManagedLedgerException exception, Object ctx) { + failedSubscriptionNames.add(subscriptionName); + clearSubscriptionResultIfNewer(subscriptionName, dataVersion); + future.complete(null); + } + }, null); + return future; + } + + private boolean isSubscriptionOldPositionInfoReadResultStillValid(String subscriptionName, + PersistentSubscription subscription, + ManagedCursor cursor, + Position markDeletePosition, + long dataVersion, + boolean preciseTimeBasedBacklogQuotaCheck) { + PersistentSubscription currentSubscription = subscriptions.get(subscriptionName); + if (currentSubscription != subscription || currentSubscription.getCursor() != cursor) { + clearSubscriptionResultIfNewer(subscriptionName, dataVersion); + return false; + } + Position currentMarkDeletePosition = cursor.getMarkDeletedPosition(); + if (currentMarkDeletePosition == null) { + clearSubscriptionResultIfNewer(subscriptionName, dataVersion); + return false; + } + if (currentMarkDeletePosition.compareTo(markDeletePosition) != 0) { + clearSubscriptionResultIfNewer(subscriptionName, dataVersion); + return false; + } + if (subscription.getNumberOfEntriesInBacklog(preciseTimeBasedBacklogQuotaCheck) <= 0) { + clearSubscriptionResultIfNewer(subscriptionName, dataVersion); + return false; + } + return true; + } + + public CompletableFuture updateSubscriptionOldPositionInfo() { + if (!brokerService.pulsar().getConfiguration().isExposeSubscriptionBacklogAgeInPrometheus()) { + clearSubscriptionOldestPositionInfos(); + return CompletableFuture.completedFuture(null); + } + + if (!(ledger.getCursors() instanceof ManagedCursorContainer managedCursorContainer)) { + // TODO: support this method with a customized managed ledger implementation + return CompletableFuture.completedFuture(null); + } + + boolean preciseTimeBasedBacklogQuotaCheck = + brokerService.pulsar().getConfiguration().isPreciseTimeBasedBacklogQuotaCheck(); + if (!hasBacklogs(preciseTimeBasedBacklogQuotaCheck)) { + clearSubscriptionOldestPositionInfos(); + return CompletableFuture.completedFuture(null); + } + return updateSubscriptionOldPositionInfos(managedCursorContainer, preciseTimeBasedBacklogQuotaCheck); + } + public CompletableFuture updateOldPositionInfo() { TopicName topicName = TopicName.get(getName()); @@ -3886,7 +4133,9 @@ public CompletableFuture updateOldPositionInfo() { return CompletableFuture.completedFuture(null); } - if (!hasBacklogs(brokerService.pulsar().getConfiguration().isPreciseTimeBasedBacklogQuotaCheck())) { + boolean preciseTimeBasedBacklogQuotaCheck = + brokerService.pulsar().getConfiguration().isPreciseTimeBasedBacklogQuotaCheck(); + if (!hasBacklogs(preciseTimeBasedBacklogQuotaCheck)) { if (log.isDebugEnabled()) { log.debug("[{}] No backlog. Update old position info is null", topicName); } @@ -3894,7 +4143,7 @@ public CompletableFuture updateOldPositionInfo() { return CompletableFuture.completedFuture(null); } - // If we have no durable cursor since `ledger.getCursors()` only managed durable cursors + // If the oldest-cursor heap has no durable cursor, there is no topic-level oldest backlog position. CursorInfo oldestMarkDeleteCursorInfo = managedCursorContainer.getCursorWithOldestPosition(); if (oldestMarkDeleteCursorInfo == null || oldestMarkDeleteCursorInfo.getPosition() == null) { if (log.isDebugEnabled()) { @@ -3927,7 +4176,7 @@ public CompletableFuture updateOldPositionInfo() { } return CompletableFuture.completedFuture(null); } - if (brokerService.pulsar().getConfiguration().isPreciseTimeBasedBacklogQuotaCheck()) { + if (preciseTimeBasedBacklogQuotaCheck) { CompletableFuture future = new CompletableFuture<>(); // Check if first unconsumed message(first message after mark delete position) // for slowest cursor's has expired. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedSubscriptionStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedSubscriptionStats.java index ca03e97c8396d..aa6a710341474 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedSubscriptionStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedSubscriptionStats.java @@ -29,6 +29,8 @@ public class AggregatedSubscriptionStats { public long msgBacklogNoDelayed; + public long backlogAgeSeconds = -1; + public boolean blockedSubscriptionOnUnackedMsgs; public double msgRateRedeliver; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java index 1531672bef1dd..45156985de407 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java @@ -83,6 +83,8 @@ public static void generate(PulsarService pulsar, boolean includeTopicMetrics, b Optional compactorMXBean = getCompactorMXBean(pulsar); LongAdder topicsCount = new LongAdder(); Map localNamespaceTopicCount = new HashMap<>(); + boolean exposeSubscriptionBacklogAge = + pulsar.getConfiguration().isExposeSubscriptionBacklogAgeInPrometheus(); pulsar.getBrokerService().getMultiLayerTopicsMap().forEach((namespace, bundlesMap) -> { namespaceStats.reset(); topicsCount.reset(); @@ -99,7 +101,7 @@ public static void generate(PulsarService pulsar, boolean includeTopicMetrics, b if (includeTopicMetrics) { topicsCount.add(1); TopicStats.printTopicStats(stream, topicStats, compactorMXBean, cluster, namespace, name, - splitTopicAndPartitionIndexLabel); + splitTopicAndPartitionIndexLabel, exposeSubscriptionBacklogAge); } else { namespaceStats.updateStats(topicStats); } @@ -133,6 +135,7 @@ private static void aggregateTopicStats(TopicStats stats, SubscriptionStatsImpl subsStats.bytesOutCounter = subscriptionStats.bytesOutCounter; subsStats.msgOutCounter = subscriptionStats.msgOutCounter; subsStats.msgBacklog = subscriptionStats.msgBacklog; + subsStats.backlogAgeSeconds = subscriptionStats.oldestBacklogMessageAgeSeconds; subsStats.msgDelayed = subscriptionStats.msgDelayed; subsStats.msgInReplay = subscriptionStats.msgInReplay; subsStats.msgRateExpired = subscriptionStats.msgRateExpired; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java index 8b208e85514a2..d496db0c4ba47 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java @@ -147,7 +147,8 @@ public void reset() { @SuppressWarnings("OptionalUsedAsFieldOrParameterType") public static void printTopicStats(PrometheusMetricStreams stream, TopicStats stats, Optional compactorMXBean, String cluster, String namespace, - String topic, boolean splitTopicAndPartitionIndexLabel) { + String topic, boolean splitTopicAndPartitionIndexLabel, + boolean exposeSubscriptionBacklogAge) { writeMetric(stream, "pulsar_subscriptions_count", stats.subscriptionsCount, cluster, namespace, topic, splitTopicAndPartitionIndexLabel); writeMetric(stream, "pulsar_producers_count", stats.producersCount, @@ -308,6 +309,10 @@ public static void printTopicStats(PrometheusMetricStreams stream, TopicStats st cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); writeSubscriptionMetric(stream, "pulsar_subscription_back_log_no_delayed", subsStats.msgBacklogNoDelayed, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + if (exposeSubscriptionBacklogAge && subsStats.backlogAgeSeconds >= 0) { + writeSubscriptionMetric(stream, "pulsar_subscription_storage_backlog_age_seconds", + subsStats.backlogAgeSeconds, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); + } writeSubscriptionMetric(stream, "pulsar_subscription_delayed", subsStats.msgDelayed, cluster, namespace, topic, sub, splitTopicAndPartitionIndexLabel); writeSubscriptionMetric(stream, "pulsar_subscription_in_replay", diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java index ccecf37486d3b..1dda4a4133ccc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java @@ -41,6 +41,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.UUID; import java.util.concurrent.CountDownLatch; @@ -78,6 +79,7 @@ import org.apache.pulsar.common.policies.data.BacklogQuota; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; +import org.apache.pulsar.common.policies.data.SubscriptionStats; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.common.policies.data.TopicType; @@ -202,14 +204,28 @@ void shutdown() throws Exception { } @BeforeMethod(alwaysRun = true) - void createNamespaces() throws PulsarAdminException { + void createNamespaces() throws Exception { config.setPreciseTimeBasedBacklogQuotaCheck(false); - admin.namespaces().createNamespace("prop/ns-quota"); - admin.namespaces().setNamespaceReplicationClusters("prop/ns-quota", Sets.newHashSet("usc")); - admin.namespaces().createNamespace("prop/quotahold"); - admin.namespaces().setNamespaceReplicationClusters("prop/quotahold", Sets.newHashSet("usc")); - admin.namespaces().createNamespace("prop/quotaholdasync"); - admin.namespaces().setNamespaceReplicationClusters("prop/quotaholdasync", Sets.newHashSet("usc")); + config.setExposeSubscriptionBacklogAgeInPrometheus(false); + createNamespaceForTest("prop/ns-quota"); + createNamespaceForTest("prop/quotahold"); + createNamespaceForTest("prop/quotaholdasync"); + } + + /** + * If a previous test's @AfterMethod timed out before the namespace was fully removed, the + * leftover would otherwise cascade as HTTP 409 here and fail every subsequent test. + * Force-delete and retry so each test starts with clean namespace state. + */ + private void createNamespaceForTest(String ns) throws Exception { + try { + admin.namespaces().createNamespace(ns); + } catch (PulsarAdminException.ConflictException e) { + log.warn("Namespace {} already exists from previous test, force-deleting and recreating", ns); + deleteNamespaceWithRetry(ns, true); + admin.namespaces().createNamespace(ns); + } + admin.namespaces().setNamespaceReplicationClusters(ns, Sets.newHashSet("usc")); } @AfterMethod(alwaysRun = true) @@ -314,6 +330,13 @@ private TopicStats getTopicStats(String topic1, boolean getPreciseBacklog) throw return stats; } + private void refreshSubscriptionBacklogAge(String topic) throws Exception { + PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic).get(); + topicRef.updateSubscriptionOldPositionInfo().get(5, SECONDS); + } + + @SuppressWarnings("deprecation") + @Test public void testTriggerBacklogQuotaSizeWithReader() throws Exception { assertEquals(admin.namespaces().getBacklogQuotaMap("prop/ns-quota"), @@ -385,8 +408,9 @@ public void testTriggerBacklogQuotaSizeWithReader() throws Exception { } @Test - public void backlogsStatsPrecise() throws PulsarAdminException, PulsarClientException, InterruptedException { + public void backlogsStatsPrecise() throws Exception { config.setPreciseTimeBasedBacklogQuotaCheck(true); + config.setExposeSubscriptionBacklogAgeInPrometheus(true); final String namespace = "prop/ns-quota"; assertEquals(admin.namespaces().getBacklogQuotaMap(namespace), new HashMap<>()); final int sizeLimitBytes = 15 * 1024 * 1024; @@ -443,6 +467,7 @@ public void backlogsStatsPrecise() throws PulsarAdminException, PulsarClientExce c1MarkDeletePositionBefore = waitForMarkDeletePositionToChange(topic1, subName1, c1MarkDeletePositionBefore); waitForQuotaCheckToRunTwice(); + refreshSubscriptionBacklogAge(topic1); Metrics metrics = prometheusMetricsClient.getMetrics(); TopicStats topicStats = getTopicStats(topic1); @@ -454,6 +479,11 @@ public void backlogsStatsPrecise() throws PulsarAdminException, PulsarClientExce assertThat(topicStats.getOldestBacklogMessageAgeSeconds()) .isCloseTo(expectedMessageAgeSeconds, within(1L)); assertThat(topicStats.getOldestBacklogMessageSubscriptionName()).isEqualTo(subName2); + assertThat(topicStats.getSubscriptions().get(subName2).getOldestBacklogMessageAgeSeconds()) + .isCloseTo(expectedMessageAgeSeconds, within(1L)); + assertThat(topicStats.getSubscriptions().get(subName1).getOldestBacklogMessageAgeSeconds()) + .isGreaterThanOrEqualTo(0L) + .isLessThan(expectedMessageAgeSeconds); Metric backlogAgeMetric = metrics.findSingleMetricByNameAndLabels("pulsar_storage_backlog_age_seconds", @@ -463,6 +493,15 @@ public void backlogsStatsPrecise() throws PulsarAdminException, PulsarClientExce entry("namespace", namespace), entry("topic", topic1)); assertThat((long) backlogAgeMetric.value).isCloseTo(expectedMessageAgeSeconds, within(2L)); + Metric subBacklogAgeMetric = + metrics.findSingleMetricByNameAndLabels("pulsar_subscription_storage_backlog_age_seconds", + Pair.of("topic", topic1), Pair.of("subscription", subName2)); + assertThat(subBacklogAgeMetric.tags).containsExactly( + entry("cluster", CLUSTER_NAME), + entry("namespace", namespace), + entry("subscription", subName2), + entry("topic", topic1)); + assertThat((long) subBacklogAgeMetric.value).isCloseTo(expectedMessageAgeSeconds, within(2L)); // Move subscription 2 away from being the oldest mark delete // S2/S1 @@ -482,6 +521,7 @@ public void backlogsStatsPrecise() throws PulsarAdminException, PulsarClientExce c1MarkDeletePositionBefore = waitForMarkDeletePositionToChange(topic1, subName1, c1MarkDeletePositionBefore); waitForQuotaCheckToRunTwice(); + refreshSubscriptionBacklogAge(topic1); metrics = prometheusMetricsClient.getMetrics(); long actualAge = (long) metrics.findByNameAndLabels( @@ -508,6 +548,7 @@ public void backlogsStatsPrecise() throws PulsarAdminException, PulsarClientExce waitForMarkDeletePositionToChange(topic1, subName1, c1MarkDeletePositionBefore); waitForQuotaCheckToRunTwice(); + refreshSubscriptionBacklogAge(topic1); // Cache shouldn't be used, since position has changed long readEntries = getReadEntries(topic1); @@ -518,8 +559,11 @@ public void backlogsStatsPrecise() throws PulsarAdminException, PulsarClientExce MILLISECONDS.toSeconds(System.currentTimeMillis() - secondOldestMessage.getPublishTime()); assertThat(topicStats.getOldestBacklogMessageAgeSeconds()).isCloseTo(expectedMessageAgeSeconds, within(2L)); assertThat(topicStats.getOldestBacklogMessageSubscriptionName()).isEqualTo(subName2); + assertThat(topicStats.getSubscriptions().get(subName2).getOldestBacklogMessageAgeSeconds()) + .isCloseTo(expectedMessageAgeSeconds, within(2L)); waitForQuotaCheckToRunTwice(); + refreshSubscriptionBacklogAge(topic1); // Cache should be used, since position hasn't changed assertThat(getReadEntries(topic1)).isEqualTo(readEntries); @@ -537,11 +581,14 @@ public void backlogsStatsPrecise() throws PulsarAdminException, PulsarClientExce log.info("Subscription 1 and 2 moved to end. Now should not backlog"); waitForMarkDeletePositionToChange(topic1, subName1, c1MarkDeletePositionBefore); waitForQuotaCheckToRunTwice(); + refreshSubscriptionBacklogAge(topic1); topicStats = getTopicStats(topic1); assertThat(topicStats.getBacklogSize()).isEqualTo(0); assertThat(topicStats.getSubscriptions().get(subName1).getMsgBacklog()).isEqualTo(0); assertThat(topicStats.getSubscriptions().get(subName2).getMsgBacklog()).isEqualTo(0); + assertThat(topicStats.getSubscriptions().get(subName1).getOldestBacklogMessageAgeSeconds()).isEqualTo(-1); + assertThat(topicStats.getSubscriptions().get(subName2).getOldestBacklogMessageAgeSeconds()).isEqualTo(-1); assertThat(topicStats.getOldestBacklogMessageAgeSeconds()).isEqualTo(-1); assertThat(topicStats.getOldestBacklogMessageSubscriptionName()).isNull(); @@ -561,6 +608,117 @@ public void backlogsStatsPrecise() throws PulsarAdminException, PulsarClientExce } } + @Test + public void subscriptionBacklogAgeStatsDisabledByDefault() throws Exception { + config.setPreciseTimeBasedBacklogQuotaCheck(true); + + try (PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()) + .statsInterval(0, SECONDS).build()) { + final String topic1 = "persistent://prop/ns-quota/topic-disabled" + UUID.randomUUID(); + final String subName = "c1"; + + client.newConsumer().topic(topic1).subscriptionName(subName) + .acknowledgmentGroupTime(0, SECONDS) + .subscribe(); + Producer producer = createProducer(client, topic1); + producer.send(new byte[1024]); + + PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic1).get(); + topicRef.updateSubscriptionOldPositionInfo().get(5, SECONDS); + + TopicStats topicStats = getTopicStats(topic1); + assertThat(topicStats.getSubscriptions().get(subName).getOldestBacklogMessageAgeSeconds()).isEqualTo(-1); + + Metrics metrics = prometheusMetricsClient.getMetrics(); + assertThat(metrics.findByNameAndLabels("pulsar_subscription_storage_backlog_age_seconds", + Pair.of("topic", topic1), Pair.of("subscription", subName))).isEmpty(); + } + } + + @Test + public void subscriptionBacklogAgeSkipsNonDurableReader() throws Exception { + config.setPreciseTimeBasedBacklogQuotaCheck(true); + config.setExposeSubscriptionBacklogAgeInPrometheus(true); + + try (PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()) + .statsInterval(0, SECONDS).build()) { + final String topic1 = "persistent://prop/ns-quota/topic-reader" + UUID.randomUUID(); + final String subName = "c1"; + + client.newConsumer().topic(topic1).subscriptionName(subName) + .acknowledgmentGroupTime(0, SECONDS) + .subscribe(); + client.newReader().topic(topic1).startMessageId(MessageId.earliest).create(); + Producer producer = createProducer(client, topic1); + producer.send(new byte[1024]); + + PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic1).get(); + topicRef.updateSubscriptionOldPositionInfo().get(5, SECONDS); + + TopicStats topicStats = getTopicStats(topic1); + assertThat(topicStats.getSubscriptions().get(subName).getOldestBacklogMessageAgeSeconds()) + .isGreaterThanOrEqualTo(0L); + + Map.Entry readerStats = topicStats.getSubscriptions().entrySet() + .stream() + .filter(entry -> !entry.getValue().isDurable()) + .findFirst() + .orElseThrow(); + assertThat(readerStats.getValue().getOldestBacklogMessageAgeSeconds()).isEqualTo(-1); + + Metrics metrics = prometheusMetricsClient.getMetrics(); + assertThat(metrics.findByNameAndLabels("pulsar_subscription_storage_backlog_age_seconds", + Pair.of("topic", topic1), Pair.of("subscription", readerStats.getKey()))).isEmpty(); + } + } + + @Test + public void subscriptionBacklogAgeCacheRemovedAfterSubscriptionDeleteAndRecreate() throws Exception { + config.setPreciseTimeBasedBacklogQuotaCheck(true); + config.setExposeSubscriptionBacklogAgeInPrometheus(true); + + try (PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()) + .statsInterval(0, SECONDS).build()) { + final String topic1 = "persistent://prop/ns-quota/topic-recreated" + UUID.randomUUID(); + final String subName1 = "c1"; + final String subName2 = "c2"; + + Consumer consumer1 = client.newConsumer().topic(topic1).subscriptionName(subName1) + .acknowledgmentGroupTime(0, SECONDS) + .subscribe(); + client.newConsumer().topic(topic1).subscriptionName(subName2) + .acknowledgmentGroupTime(0, SECONDS) + .subscribe(); + Producer producer = createProducer(client, topic1); + producer.send(new byte[1024]); + + PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic1).get(); + topicRef.updateSubscriptionOldPositionInfo().get(5, SECONDS); + + TopicStats topicStats = getTopicStats(topic1); + assertThat(topicStats.getSubscriptions().get(subName1).getOldestBacklogMessageAgeSeconds()) + .isGreaterThanOrEqualTo(0L); + assertThat(topicStats.getSubscriptions().get(subName2).getOldestBacklogMessageAgeSeconds()) + .isGreaterThanOrEqualTo(0L); + + consumer1.unsubscribe(); + topicRef.updateSubscriptionOldPositionInfo().get(5, SECONDS); + + client.newConsumer().topic(topic1).subscriptionName(subName1) + .acknowledgmentGroupTime(0, SECONDS) + .subscribe(); + topicRef.updateSubscriptionOldPositionInfo().get(5, SECONDS); + + topicStats = getTopicStats(topic1); + assertThat(topicStats.getSubscriptions().get(subName1).getMsgBacklog()).isEqualTo(0); + assertThat(topicStats.getSubscriptions().get(subName1).getOldestBacklogMessageAgeSeconds()).isEqualTo(-1); + assertThat(topicStats.getSubscriptions().get(subName2).getOldestBacklogMessageAgeSeconds()) + .isGreaterThanOrEqualTo(0L); + } + } + + @SuppressWarnings("deprecation") + @Test public void backlogsStatsPreciseWithNoBacklog() throws PulsarAdminException, PulsarClientException, InterruptedException { @@ -782,8 +940,9 @@ private long getReadEntries(String topic1) { } @Test - public void backlogsStatsNotPrecise() throws PulsarAdminException, PulsarClientException, InterruptedException { + public void backlogsStatsNotPrecise() throws Exception { config.setPreciseTimeBasedBacklogQuotaCheck(false); + config.setExposeSubscriptionBacklogAgeInPrometheus(true); config.setManagedLedgerMaxEntriesPerLedger(6); final String namespace = "prop/ns-quota"; assertEquals(admin.namespaces().getBacklogQuotaMap(namespace), new HashMap<>()); @@ -852,12 +1011,15 @@ public void backlogsStatsNotPrecise() throws PulsarAdminException, PulsarClientE long unloadTime = System.currentTimeMillis(); waitForQuotaCheckToRunTwice(); + refreshSubscriptionBacklogAge(topic1); topicStats = getTopicStats(topic1); assertThat(topicStats.getOldestBacklogMessageSubscriptionName()).isEqualTo(subName2); // age is measured against the ledger closing time long expectedAge = MILLISECONDS.toSeconds(System.currentTimeMillis() - unloadTime); assertThat(topicStats.getOldestBacklogMessageAgeSeconds()).isCloseTo(expectedAge, within(1L)); + assertThat(topicStats.getSubscriptions().get(subName2).getOldestBacklogMessageAgeSeconds()) + .isCloseTo(expectedAge, within(1L)); String c2MarkDeletePositionBefore = admin.topics().getInternalStats(topic1).cursors.get(subName2).markDeletePosition; @@ -872,11 +1034,14 @@ public void backlogsStatsNotPrecise() throws PulsarAdminException, PulsarClientE waitForMarkDeletePositionToChange(topic1, subName1, c1MarkDeletePositionBefore); waitForMarkDeletePositionToChange(topic1, subName2, c2MarkDeletePositionBefore); waitForQuotaCheckToRunTwice(); + refreshSubscriptionBacklogAge(topic1); topicStats = getTopicStats(topic1); assertThat(topicStats.getOldestBacklogMessageSubscriptionName()).isEqualTo(subName2); expectedAge = MILLISECONDS.toSeconds(System.currentTimeMillis() - unloadTime); assertThat(topicStats.getOldestBacklogMessageAgeSeconds()).isCloseTo(expectedAge, within(1L)); + assertThat(topicStats.getSubscriptions().get(subName2).getOldestBacklogMessageAgeSeconds()) + .isCloseTo(expectedAge, within(1L)); // Unsubscribe consume1 and consumer2 consumer1.unsubscribe(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregatorTest.java index e51d6a886e010..25c926724f0d1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregatorTest.java @@ -108,7 +108,10 @@ public void testGenerateSubscriptionsStats() { PrometheusMetricStreams metricStreams = Mockito.spy(new PrometheusMetricStreams()); // Populate subscriptions stats + ServiceConfiguration config = pulsar.getConfiguration(); + doReturn(true).when(config).isExposeSubscriptionBacklogAgeInPrometheus(); subStats.blockedSubscriptionOnUnackedMsgs = true; + subStats.oldestBacklogMessageAgeSeconds = 123; consumerStats.blockedConsumerOnUnackedMsgs = false; // should not affect blockedSubscriptionOnUnackedMsgs consumerStats.unackedMessages = 1; consumerStats.msgRateRedeliver = 0.7; @@ -124,6 +127,7 @@ public void testGenerateSubscriptionsStats() { verifySubscriptionMetric(metricStreams, "pulsar_subscription_msg_rate_redeliver", 0.7); verifySubscriptionMetric(metricStreams, "pulsar_subscription_unacked_messages", 1L); + verifySubscriptionMetric(metricStreams, "pulsar_subscription_storage_backlog_age_seconds", 123L); } private void verifySubscriptionMetric(PrometheusMetricStreams metricStreams, String metricName, Number value) { diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SubscriptionStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SubscriptionStats.java index 05cfd55b0456c..aa26c8bc40d9a 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SubscriptionStats.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/SubscriptionStats.java @@ -57,6 +57,18 @@ public interface SubscriptionStats { /** Get the publish time of the earliest message in the backlog. */ long getEarliestMsgPublishTimeInBacklog(); + /** + * Age of oldest unacknowledged message for this subscription, in seconds. + *

+ * This is a best-effort cached value from the broker's periodic subscription backlog-age refresh. The value is + * {@code -1} when it is unknown, not applicable, the subscription has no backlog, or the broker has disabled + * subscription backlog-age computation. + *

+ */ + default long getOldestBacklogMessageAgeSeconds() { + return -1; + } + /** Number of entries in the subscription backlog that do not contain the delay messages. */ long getMsgBacklogNoDelayed(); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java index df91798f48737..ee67c197daae2 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java @@ -65,6 +65,16 @@ public class SubscriptionStatsImpl implements SubscriptionStats { /** Get the publish time of the earliest message in the backlog. */ public long earliestMsgPublishTimeInBacklog; + /** + * Age of oldest unacknowledged message for this subscription, in seconds. + *

+ * This is a best-effort cached value from the broker's periodic subscription backlog-age refresh. The value is + * {@code -1} when it is unknown, not applicable, the subscription has no backlog, or the broker has disabled + * subscription backlog-age computation. + *

+ */ + public long oldestBacklogMessageAgeSeconds = -1; + /** Number of entries in the subscription backlog that do not contain the delay messages. */ public long msgBacklogNoDelayed; @@ -220,6 +230,7 @@ public void reset() { nonContiguousDeletedMessagesRanges = 0; nonContiguousDeletedMessagesRangesSerializedSize = 0; earliestMsgPublishTimeInBacklog = 0L; + oldestBacklogMessageAgeSeconds = -1; delayedMessageIndexSizeInBytes = 0; subscriptionProperties.clear(); filterProcessedMsgCount = 0; @@ -285,6 +296,8 @@ public SubscriptionStatsImpl add(SubscriptionStatsImpl stats) { stats.earliestMsgPublishTimeInBacklog ); } + this.oldestBacklogMessageAgeSeconds = Math.max( + this.oldestBacklogMessageAgeSeconds, stats.oldestBacklogMessageAgeSeconds); this.delayedMessageIndexSizeInBytes += stats.delayedMessageIndexSizeInBytes; this.subscriptionProperties.putAll(stats.subscriptionProperties); this.filterProcessedMsgCount += stats.filterProcessedMsgCount; diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImplTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImplTest.java index 8a4b5da9edd20..d6c3222d2b01a 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImplTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImplTest.java @@ -27,8 +27,10 @@ public class SubscriptionStatsImplTest { public void testReset() { SubscriptionStatsImpl stats = new SubscriptionStatsImpl(); stats.earliestMsgPublishTimeInBacklog = 1L; + stats.oldestBacklogMessageAgeSeconds = 10L; stats.reset(); assertEquals(stats.earliestMsgPublishTimeInBacklog, 0L); + assertEquals(stats.oldestBacklogMessageAgeSeconds, -1L); } @@ -79,4 +81,19 @@ public void testAdd_EarliestMsgPublishTimeInBacklogs_Zero() { SubscriptionStatsImpl aggregate = stats1.add(stats2); assertEquals(aggregate.earliestMsgPublishTimeInBacklog, 0L); } -} \ No newline at end of file + + @Test + public void testAdd_OldestBacklogMessageAgeSeconds() { + SubscriptionStatsImpl stats1 = new SubscriptionStatsImpl(); + stats1.oldestBacklogMessageAgeSeconds = -1L; + + SubscriptionStatsImpl stats2 = new SubscriptionStatsImpl(); + stats2.oldestBacklogMessageAgeSeconds = 20L; + + SubscriptionStatsImpl stats3 = new SubscriptionStatsImpl(); + stats3.oldestBacklogMessageAgeSeconds = 10L; + + SubscriptionStatsImpl aggregate = stats1.add(stats2).add(stats3); + assertEquals(aggregate.oldestBacklogMessageAgeSeconds, 20L); + } +} From 43c03dde1e22a548326f59b2a33f38be75f0d090 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Thu, 27 Aug 2026 12:16:58 +0800 Subject: [PATCH 190/213] Fix cherry-pick Signed-off-by: Zixuan Liu --- .../pulsar/broker/service/BrokerService.java | 4 +- .../broker/service/TopicLoadingContext.java | 12 +++-- .../broker/service/BrokerServiceTest.java | 52 ++++++++++++------- .../PersistentSubscriptionTest.java | 1 - 4 files changed, 45 insertions(+), 24 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index d90b0c48674e6..d6d4ac88e4cff 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -1241,8 +1241,8 @@ public CompletableFuture> getTopic(TopicLoadingContext context) "Broker is unable to load persistent topic")); } + final var timeoutSeconds = pulsar.getConfiguration().getTopicLoadTimeoutSeconds(); if (context.getTopicFuture() == null) { - final var timeoutSeconds = pulsar.getConfiguration().getTopicLoadTimeoutSeconds(); final CompletableFuture> topicFuture = FutureUtil.createFutureWithTimeout( Duration.ofSeconds(timeoutSeconds), executor(), () -> FAILED_TO_LOAD_TOPIC_TIMEOUT_EXCEPTION); @@ -3491,7 +3491,7 @@ private void createPendingLoadTopic() { return; } - pendingTopic.polledFromQueue(); + pendingTopic.trace("queued"); final String topic = pendingTopic.getTopicName().toString(); checkTopicNsOwnership(topic).thenRun(() -> { CompletableFuture> pendingFuture = pendingTopic.getTopicFuture(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicLoadingContext.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicLoadingContext.java index dcba755ae0e07..cb9ab3f534074 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicLoadingContext.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicLoadingContext.java @@ -22,13 +22,13 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedQueue; +import lombok.Builder; import lombok.Getter; import lombok.Setter; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.util.LatencyTracer; import org.jspecify.annotations.Nullable; -@Builder public class TopicLoadingContext extends LatencyTracer { @Getter @@ -38,7 +38,7 @@ public class TopicLoadingContext extends LatencyTracer { private boolean createIfMissing; @Getter @Setter - private CompletableFuture> topicFuture; + private CompletableFuture> topicFuture; @Getter @Setter @Nullable private Map properties; @@ -50,12 +50,18 @@ public class TopicLoadingContext extends LatencyTracer { @Setter private String proxyVersion; + @Builder public TopicLoadingContext(TopicName topicName, boolean createIfMissing, - CompletableFuture> topicFuture) { + CompletableFuture> topicFuture, + @Nullable Map properties, String clientVersion, + String proxyVersion) { // The topic loading could be ended asynchronously by a timeout event, so we need a thread safe queue here super(new ConcurrentLinkedQueue<>(), System::nanoTime); this.topicName = topicName; this.createIfMissing = createIfMissing; this.topicFuture = topicFuture; + this.properties = properties; + this.clientVersion = clientVersion; + this.proxyVersion = proxyVersion; } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java index 0d0e32f0d4f90..9df76f4a2bf2a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java @@ -1347,9 +1347,13 @@ public void testCleanUnloadedTopicFromCacheDoesNotRemoveNewTopicFuture() throws CompletableFuture> newTopicFuture = CompletableFuture.completedFuture(Optional.of(mock(Topic.class))); List unloadEvents = new ArrayList<>(); - TopicEventsListener listener = (name, event, stage, t) -> { - if (topicName.equals(name) && event == TopicEventsListener.TopicEvent.UNLOAD) { - unloadEvents.add(stage); + TopicEventsListener listener = new TopicEventsListener() { + @Override + public void handleEvent(String name, TopicEventsListener.TopicEvent event, + TopicEventsListener.EventStage stage, Throwable t) { + if (topicName.equals(name) && event == TopicEventsListener.TopicEvent.UNLOAD) { + unloadEvents.add(stage); + } } }; brokerService.addTopicEventListener(listener); @@ -1389,10 +1393,14 @@ public void testUnloadBeforeDoesNotStartTopicReload() throws Exception { assertNotNull(topicFuture); AtomicReference>> futureAtUnloadBefore = new AtomicReference<>(); - TopicEventsListener listener = (name, event, stage, t) -> { - if (topicName.equals(name) && event == TopicEventsListener.TopicEvent.UNLOAD) { - if (stage == TopicEventsListener.EventStage.BEFORE) { - futureAtUnloadBefore.set(brokerService.getTopic(TopicName.get(topicName), true, null)); + TopicEventsListener listener = new TopicEventsListener() { + @Override + public void handleEvent(String name, TopicEventsListener.TopicEvent event, + TopicEventsListener.EventStage stage, Throwable t) { + if (topicName.equals(name) && event == TopicEventsListener.TopicEvent.UNLOAD) { + if (stage == TopicEventsListener.EventStage.BEFORE) { + futureAtUnloadBefore.set(brokerService.getTopic(TopicName.get(topicName), true, null)); + } } } }; @@ -1421,14 +1429,18 @@ public void testRemoveTopicFromCacheIgnoresReentrantUnloadCallback() throws Exce List unloadEvents = new ArrayList<>(); AtomicReference reentrantCallbackError = new AtomicReference<>(); - TopicEventsListener listener = (name, event, stage, t) -> { - if (topicName.equals(name) && event == TopicEventsListener.TopicEvent.UNLOAD) { - unloadEvents.add(stage); - if (stage == TopicEventsListener.EventStage.BEFORE) { - try { - brokerService.removeTopicFromCache((AbstractTopic) topic).get(5, TimeUnit.SECONDS); - } catch (Throwable e) { - reentrantCallbackError.set(e); + TopicEventsListener listener = new TopicEventsListener() { + @Override + public void handleEvent(String name, TopicEventsListener.TopicEvent event, + TopicEventsListener.EventStage stage, Throwable t) { + if (topicName.equals(name) && event == TopicEventsListener.TopicEvent.UNLOAD) { + unloadEvents.add(stage); + if (stage == TopicEventsListener.EventStage.BEFORE) { + try { + brokerService.removeTopicFromCache((AbstractTopic) topic).get(5, TimeUnit.SECONDS); + } catch (Throwable e) { + reentrantCallbackError.set(e); + } } } } @@ -1459,9 +1471,13 @@ public void testRemoveTopicFromCacheDoesNotRemoveSupersededTopicFuture() throws assertNotNull(oldTopicFuture); List unloadEvents = new ArrayList<>(); - TopicEventsListener listener = (name, event, stage, t) -> { - if (topicName.equals(name) && event == TopicEventsListener.TopicEvent.UNLOAD) { - unloadEvents.add(stage); + TopicEventsListener listener = new TopicEventsListener() { + @Override + public void handleEvent(String name, TopicEventsListener.TopicEvent event, + TopicEventsListener.EventStage stage, Throwable t) { + if (topicName.equals(name) && event == TopicEventsListener.TopicEvent.UNLOAD) { + unloadEvents.add(stage); + } } }; brokerService.addTopicEventListener(listener); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java index c73ead2f6eaf9..6fe5675c94461 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.broker.service.persistent; -import static org.assertj.core.api.Assertions.assertThat; import static org.apache.bookkeeper.mledger.ManagedCursor.CURSOR_INTERNAL_PROPERTY_PREFIX; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; From 8afddf0332ca5d20b5ce79b81427b58619fbb24e Mon Sep 17 00:00:00 2001 From: sinan liu Date: Tue, 7 Apr 2026 11:33:22 +0800 Subject: [PATCH 191/213] [refactor][broker] Decouple delayed delivery trackers from dispatcher (#25384) --- ...BucketDelayedDeliveryTrackerBenchmark.java | 321 ++++++++++++++ ...DelayedDeliveryTrackerSimpleBenchmark.java | 408 ------------------ .../bucket/MockBucketSnapshotStorage.java | 106 +++++ .../AbstractDelayedDeliveryTracker.java | 31 +- .../delayed/DelayedDeliveryContext.java | 32 ++ .../DispatcherDelayedDeliveryContext.java | 51 +++ .../InMemoryDelayedDeliveryTracker.java | 19 +- .../delayed/NoopDelayedDeliveryContext.java | 53 +++ .../bucket/BucketDelayedDeliveryTracker.java | 67 +-- 9 files changed, 641 insertions(+), 447 deletions(-) create mode 100644 microbench/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerBenchmark.java delete mode 100644 microbench/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerSimpleBenchmark.java create mode 100644 microbench/src/main/java/org/apache/pulsar/broker/delayed/bucket/MockBucketSnapshotStorage.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryContext.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DispatcherDelayedDeliveryContext.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/NoopDelayedDeliveryContext.java diff --git a/microbench/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerBenchmark.java b/microbench/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerBenchmark.java new file mode 100644 index 0000000000000..08d02195c8781 --- /dev/null +++ b/microbench/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerBenchmark.java @@ -0,0 +1,321 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.delayed.bucket; + +import io.netty.util.HashedWheelTimer; +import io.netty.util.Timer; +import io.netty.util.concurrent.DefaultThreadFactory; +import java.time.Clock; +import java.util.NavigableSet; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.bookkeeper.mledger.impl.ActiveManagedCursorContainerImpl; +import org.apache.bookkeeper.mledger.impl.MockManagedCursor; +import org.apache.pulsar.broker.delayed.DelayedDeliveryTracker; +import org.apache.pulsar.broker.delayed.NoopDelayedDeliveryContext; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; + +/** + * JMH benchmarks for {@link BucketDelayedDeliveryTracker}. + * + *

This benchmark measures tracker throughput under different read/write ratios + * and initial message counts without implying a specific lock implementation. + * + *

Run with: mvn exec:java -Dexec.mainClass="org.openjdk.jmh.Main" + * -Dexec.args="BucketDelayedDeliveryTrackerBenchmark" + */ +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) +@State(Scope.Benchmark) +@Warmup(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1) +@Measurement(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1) +@Fork(1) +public class BucketDelayedDeliveryTrackerBenchmark { + + /** + * Fixed delivery timestamp base that stays far beyond any benchmark trial duration, + * so scheduled tasks will not start firing while throughput is being measured. + */ + private static final long FUTURE_DELIVERY_BASE_TIME_MILLIS = 4102444800000L; // 2100-01-01T00:00:00Z + + @Param({"90_10", "80_20", "70_30", "50_50"}) + public String readWriteRatio; + + @Param({"1000", "5000", "8000"}) + public int initialMessages; + + private BucketDelayedDeliveryTracker tracker; + private Timer timer; + private MockBucketSnapshotStorage storage; + private NoopDelayedDeliveryContext context; + private AtomicLong messageIdGenerator; + private int readPercentage; + private long futureDeliveryBaseTimeMillis; + /** + * Maximum number of additional unique (ledgerId, entryId) positions to + * introduce per trial on top of {@link #initialMessages}. This allows + * controlling the memory footprint of the benchmark while still applying + * sustained write pressure to the tracker. + * + *

Use {@code -p maxAdditionalUniqueMessages=...} on the JMH command line + * to tune the load. The default value is conservative for local runs.

+ */ + @Param({"1000000"}) + public long maxAdditionalUniqueMessages; + /** + * Upper bound on the absolute message id that will be used to derive + * (ledgerId, entryId) positions during a single trial. + */ + private long maxUniqueMessageId; + /** + * In real Pulsar usage, {@link DelayedDeliveryTracker#addMessage(long, long, long)} is invoked + * by a single dispatcher thread and messages arrive in order of (ledgerId, entryId). + *

+ * To reflect this invariant in the benchmark, all write operations that end up calling + * {@code tracker.addMessage(...)} are serialized via this mutex so that the tracker only + * ever observes a single writer with monotonically increasing ids, even when JMH runs the + * benchmark method with multiple threads. + */ + private final Object writeMutex = new Object(); + + @Setup(Level.Trial) + public void setup() throws Exception { + setupMockComponents(); + createTracker(); + String[] parts = readWriteRatio.split("_"); + readPercentage = Integer.parseInt(parts[0]); + futureDeliveryBaseTimeMillis = FUTURE_DELIVERY_BASE_TIME_MILLIS; + preloadMessages(); + messageIdGenerator = new AtomicLong(initialMessages + 1); + // Allow a bounded number of additional unique messages per trial to avoid + // unbounded memory growth while still stressing the indexing logic. + maxUniqueMessageId = initialMessages + maxAdditionalUniqueMessages; + } + + @TearDown(Level.Trial) + public void tearDown() throws Exception { + if (tracker != null) { + tracker.close(); + } + if (timer != null) { + timer.stop(); + } + } + + private void setupMockComponents() throws Exception { + timer = new HashedWheelTimer(new DefaultThreadFactory("test-delayed-delivery"), 100, TimeUnit.MILLISECONDS); + storage = new MockBucketSnapshotStorage(); + + ActiveManagedCursorContainerImpl container = new ActiveManagedCursorContainerImpl(); + MockManagedCursor cursor = MockManagedCursor.createCursor(container, "test-cursor", + PositionFactory.create(0, 0)); + // Use the same " / " naming pattern as real dispatchers, + // so that Bucket.asyncSaveBucketSnapshot can correctly derive topicName. + String dispatcherName = "persistent://public/default/jmh-topic / " + cursor.getName(); + context = new NoopDelayedDeliveryContext(dispatcherName, cursor); + } + + private void createTracker() throws Exception { + tracker = new BucketDelayedDeliveryTracker( + context, timer, 1000, Clock.systemUTC(), true, storage, + 20, 1000, 100, 50 + ); + } + + private void preloadMessages() { + // Preload messages to create realistic test conditions while keeping + // delivery timestamps far beyond the benchmark trial duration so the + // tracker's timer does not start firing during measurement. + long baseTime = futureDeliveryBaseTimeMillis; + for (int i = 1; i <= initialMessages; i++) { + tracker.addMessage(i, i, baseTime + i * 1000L); + } + } + + // ============================================================================= + // READ-WRITE RATIO BENCHMARKS + // ============================================================================= + + @Benchmark + public boolean benchmarkMixedOperations() { + if (ThreadLocalRandom.current().nextInt(100) < readPercentage) { + // Read operations + return performReadOperation(); + } else { + // Write operations + return performWriteOperation(); + } + } + + /** + * Serialize calls to {@link BucketDelayedDeliveryTracker#addMessage(long, long, long)} and + * ensure (ledgerId, entryId) are generated in a strictly increasing sequence, matching the + * real dispatcher single-threaded behaviour. + */ + private boolean addMessageSequential(long deliverAt, int entryIdModulo) { + synchronized (writeMutex) { + long id = messageIdGenerator.getAndIncrement(); + // Limit the number of distinct positions that are introduced into the tracker + // to keep memory usage bounded. Once the upper bound is reached, we re-use + // the last position id so that subsequent calls behave like updates to + // existing messages and are short-circuited by containsMessage checks. + long boundedId = Math.min(id, maxUniqueMessageId); + long ledgerId = boundedId; + long entryId = boundedId % entryIdModulo; + return tracker.addMessage(ledgerId, entryId, deliverAt); + } + } + + private boolean performReadOperation() { + int operation = ThreadLocalRandom.current().nextInt(3); + switch (operation) { + case 0: + // containsMessage + long ledgerId = ThreadLocalRandom.current().nextLong(1, initialMessages + 100); + long entryId = ThreadLocalRandom.current().nextLong(1, 1000); + return tracker.containsMessage(ledgerId, entryId); + case 1: + // nextDeliveryTime + try { + tracker.nextDeliveryTime(); + return true; + } catch (Exception e) { + return false; + } + case 2: + // getNumberOfDelayedMessages + long count = tracker.getNumberOfDelayedMessages(); + return count >= 0; + default: + return false; + } + } + + private boolean performWriteOperation() { + long deliverAt = futureDeliveryBaseTimeMillis + ThreadLocalRandom.current().nextLong(5000, 30000); + return addMessageSequential(deliverAt, 1000); + } + + // ============================================================================= + // SPECIFIC OPERATION BENCHMARKS + // ============================================================================= + + @Benchmark + @Threads(8) + public boolean benchmarkConcurrentContainsMessage() { + long ledgerId = ThreadLocalRandom.current().nextLong(1, initialMessages + 100); + long entryId = ThreadLocalRandom.current().nextLong(1, 1000); + return tracker.containsMessage(ledgerId, entryId); + } + + @Benchmark + @Threads(4) + public boolean benchmarkConcurrentAddMessage() { + long deliverAt = futureDeliveryBaseTimeMillis + ThreadLocalRandom.current().nextLong(10000, 60000); + return addMessageSequential(deliverAt, 1000); + } + + @Benchmark + @Threads(2) + public NavigableSet benchmarkConcurrentGetScheduledMessages() { + // Create some messages ready for delivery + long currentTime = System.currentTimeMillis(); + for (int i = 0; i < 5; i++) { + addMessageSequential(currentTime - 1000, 100); + } + return tracker.getScheduledMessages(10); + } + + @Benchmark + @Threads(16) + public long benchmarkConcurrentNextDeliveryTime() { + try { + return tracker.nextDeliveryTime(); + } catch (Exception e) { + return -1; + } + } + + @Benchmark + @Threads(1) + public long benchmarkGetNumberOfDelayedMessages() { + return tracker.getNumberOfDelayedMessages(); + } + + // ============================================================================= + // HIGH CONTENTION SCENARIOS + // ============================================================================= + + @Benchmark + @Threads(32) + public boolean benchmarkHighContentionMixedOperations() { + return benchmarkMixedOperations(); + } + + @Benchmark + @Threads(16) + public boolean benchmarkContentionReads() { + return performReadOperation(); + } + + @Benchmark + @Threads(8) + public boolean benchmarkContentionWrites() { + return performWriteOperation(); + } + + // ============================================================================= + // THROUGHPUT BENCHMARKS + // ============================================================================= + + @Benchmark + @Threads(1) + public boolean benchmarkSingleThreadedThroughput() { + return benchmarkMixedOperations(); + } + + @Benchmark + @Threads(4) + public boolean benchmarkMediumConcurrencyThroughput() { + return benchmarkMixedOperations(); + } + + @Benchmark + @Threads(8) + public boolean benchmarkHighConcurrencyThroughput() { + return benchmarkMixedOperations(); + } + +} diff --git a/microbench/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerSimpleBenchmark.java b/microbench/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerSimpleBenchmark.java deleted file mode 100644 index 985e714d54d1d..0000000000000 --- a/microbench/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerSimpleBenchmark.java +++ /dev/null @@ -1,408 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.delayed.bucket; - -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.StampedLock; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Level; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Param; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.TearDown; -import org.openjdk.jmh.annotations.Threads; -import org.openjdk.jmh.annotations.Warmup; - -/** - * Simplified JMH Benchmarks for BucketDelayedDeliveryTracker thread safety improvements. - * This benchmark focuses on the core StampedLock optimistic read performance without - * complex dependencies on the full BucketDelayedDeliveryTracker implementation. - * Run with: mvn exec:java -Dexec.mainClass="org.openjdk.jmh.Main" - * -Dexec.args="BucketDelayedDeliveryTrackerSimpleBenchmark" - */ -@BenchmarkMode(Mode.Throughput) -@OutputTimeUnit(TimeUnit.SECONDS) -@State(Scope.Benchmark) -@Warmup(iterations = 5, time = 1) -@Measurement(iterations = 5, time = 1) -@Fork(1) -public class BucketDelayedDeliveryTrackerSimpleBenchmark { - - @Param({"1", "2", "4", "8", "16"}) - public int threadCount; - - private StampedLock stampedLock; - private boolean testData = true; - private volatile long counter = 0; - - @Setup(Level.Trial) - public void setup() throws Exception { - stampedLock = new StampedLock(); - } - - @TearDown(Level.Trial) - public void tearDown() throws Exception { - // Cleanup if needed - } - - // ============================================================================= - // STAMPED LOCK OPTIMISTIC READ BENCHMARKS - // ============================================================================= - - @Benchmark - @Threads(1) - public boolean benchmarkOptimisticReadSingleThreaded() { - // Simulate optimistic read like in containsMessage() - long stamp = stampedLock.tryOptimisticRead(); - boolean result = testData; // Simulate reading shared data - - if (!stampedLock.validate(stamp)) { - stamp = stampedLock.readLock(); - try { - result = testData; - } finally { - stampedLock.unlockRead(stamp); - } - } - return result; - } - - @Benchmark - @Threads(2) - public boolean benchmarkOptimisticReadMultiThreaded() { - long stamp = stampedLock.tryOptimisticRead(); - boolean result = testData; - - if (!stampedLock.validate(stamp)) { - stamp = stampedLock.readLock(); - try { - result = testData; - } finally { - stampedLock.unlockRead(stamp); - } - } - return result; - } - - @Benchmark - @Threads(8) - public boolean benchmarkOptimisticReadHighConcurrency() { - long stamp = stampedLock.tryOptimisticRead(); - boolean result = testData; - - if (!stampedLock.validate(stamp)) { - stamp = stampedLock.readLock(); - try { - result = testData; - } finally { - stampedLock.unlockRead(stamp); - } - } - return result; - } - - @Benchmark - @Threads(16) - public boolean benchmarkOptimisticReadExtremeConcurrency() { - long stamp = stampedLock.tryOptimisticRead(); - boolean result = testData; - - if (!stampedLock.validate(stamp)) { - stamp = stampedLock.readLock(); - try { - result = testData; - } finally { - stampedLock.unlockRead(stamp); - } - } - return result; - } - - // ============================================================================= - // READ:WRITE RATIO BENCHMARKS (as requested) - // ============================================================================= - - @Benchmark - @Threads(4) - public boolean benchmarkReadWrite10_90() { - // 10:90 read:write ratio simulation - if (ThreadLocalRandom.current().nextInt(100) < 10) { - // Read operation - long stamp = stampedLock.tryOptimisticRead(); - boolean result = testData; - - if (!stampedLock.validate(stamp)) { - stamp = stampedLock.readLock(); - try { - result = testData; - } finally { - stampedLock.unlockRead(stamp); - } - } - return result; - } else { - // Write operation - long stamp = stampedLock.writeLock(); - try { - testData = !testData; - counter++; - return testData; - } finally { - stampedLock.unlockWrite(stamp); - } - } - } - - @Benchmark - @Threads(4) - public boolean benchmarkReadWrite20_80() { - // 20:80 read:write ratio - if (ThreadLocalRandom.current().nextInt(100) < 20) { - long stamp = stampedLock.tryOptimisticRead(); - boolean result = testData; - - if (!stampedLock.validate(stamp)) { - stamp = stampedLock.readLock(); - try { - result = testData; - } finally { - stampedLock.unlockRead(stamp); - } - } - return result; - } else { - long stamp = stampedLock.writeLock(); - try { - testData = !testData; - counter++; - return testData; - } finally { - stampedLock.unlockWrite(stamp); - } - } - } - - @Benchmark - @Threads(4) - public boolean benchmarkReadWrite40_60() { - // 40:60 read:write ratio - if (ThreadLocalRandom.current().nextInt(100) < 40) { - long stamp = stampedLock.tryOptimisticRead(); - boolean result = testData; - - if (!stampedLock.validate(stamp)) { - stamp = stampedLock.readLock(); - try { - result = testData; - } finally { - stampedLock.unlockRead(stamp); - } - } - return result; - } else { - long stamp = stampedLock.writeLock(); - try { - testData = !testData; - counter++; - return testData; - } finally { - stampedLock.unlockWrite(stamp); - } - } - } - - @Benchmark - @Threads(4) - public boolean benchmarkReadWrite50_50() { - // 50:50 read:write ratio - if (ThreadLocalRandom.current().nextInt(100) < 50) { - long stamp = stampedLock.tryOptimisticRead(); - boolean result = testData; - - if (!stampedLock.validate(stamp)) { - stamp = stampedLock.readLock(); - try { - result = testData; - } finally { - stampedLock.unlockRead(stamp); - } - } - return result; - } else { - long stamp = stampedLock.writeLock(); - try { - testData = !testData; - counter++; - return testData; - } finally { - stampedLock.unlockWrite(stamp); - } - } - } - - @Benchmark - @Threads(4) - public boolean benchmarkReadWrite60_40() { - // 60:40 read:write ratio - if (ThreadLocalRandom.current().nextInt(100) < 60) { - long stamp = stampedLock.tryOptimisticRead(); - boolean result = testData; - - if (!stampedLock.validate(stamp)) { - stamp = stampedLock.readLock(); - try { - result = testData; - } finally { - stampedLock.unlockRead(stamp); - } - } - return result; - } else { - long stamp = stampedLock.writeLock(); - try { - testData = !testData; - counter++; - return testData; - } finally { - stampedLock.unlockWrite(stamp); - } - } - } - - @Benchmark - @Threads(4) - public boolean benchmarkReadWrite80_20() { - // 80:20 read:write ratio - if (ThreadLocalRandom.current().nextInt(100) < 80) { - long stamp = stampedLock.tryOptimisticRead(); - boolean result = testData; - - if (!stampedLock.validate(stamp)) { - stamp = stampedLock.readLock(); - try { - result = testData; - } finally { - stampedLock.unlockRead(stamp); - } - } - return result; - } else { - long stamp = stampedLock.writeLock(); - try { - testData = !testData; - counter++; - return testData; - } finally { - stampedLock.unlockWrite(stamp); - } - } - } - - @Benchmark - @Threads(4) - public boolean benchmarkReadWrite90_10() { - // 90:10 read:write ratio - most realistic for production - if (ThreadLocalRandom.current().nextInt(100) < 90) { - long stamp = stampedLock.tryOptimisticRead(); - boolean result = testData; - - if (!stampedLock.validate(stamp)) { - stamp = stampedLock.readLock(); - try { - result = testData; - } finally { - stampedLock.unlockRead(stamp); - } - } - return result; - } else { - long stamp = stampedLock.writeLock(); - try { - testData = !testData; - counter++; - return testData; - } finally { - stampedLock.unlockWrite(stamp); - } - } - } - - // ============================================================================= - // HIGH CONCURRENCY SCENARIOS - // ============================================================================= - - @Benchmark - @Threads(8) - public boolean benchmarkReadWrite90_10_HighConcurrency() { - // 90:10 read:write ratio with high concurrency - if (ThreadLocalRandom.current().nextInt(100) < 90) { - long stamp = stampedLock.tryOptimisticRead(); - boolean result = testData; - - if (!stampedLock.validate(stamp)) { - stamp = stampedLock.readLock(); - try { - result = testData; - } finally { - stampedLock.unlockRead(stamp); - } - } - return result; - } else { - long stamp = stampedLock.writeLock(); - try { - testData = !testData; - counter++; - return testData; - } finally { - stampedLock.unlockWrite(stamp); - } - } - } - - @Benchmark - @Threads(16) - public boolean benchmarkOptimisticReadContention() { - // High contention scenario to test optimistic read fallback behavior - long stamp = stampedLock.tryOptimisticRead(); - boolean result = testData; - - // Simulate some computation - if (ThreadLocalRandom.current().nextInt(1000) == 0) { - Thread.yield(); // Occasionally yield to increase contention - } - - if (!stampedLock.validate(stamp)) { - stamp = stampedLock.readLock(); - try { - result = testData; - } finally { - stampedLock.unlockRead(stamp); - } - } - return result; - } -} \ No newline at end of file diff --git a/microbench/src/main/java/org/apache/pulsar/broker/delayed/bucket/MockBucketSnapshotStorage.java b/microbench/src/main/java/org/apache/pulsar/broker/delayed/bucket/MockBucketSnapshotStorage.java new file mode 100644 index 0000000000000..c89071d31a097 --- /dev/null +++ b/microbench/src/main/java/org/apache/pulsar/broker/delayed/bucket/MockBucketSnapshotStorage.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.delayed.bucket; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; + +public class MockBucketSnapshotStorage implements BucketSnapshotStorage { + + private final AtomicLong idGenerator = new AtomicLong(1); + private final Map snapshots = new ConcurrentHashMap<>(); + private final Map> snapshotSegments = new ConcurrentHashMap<>(); + private final Map snapshotLengths = new ConcurrentHashMap<>(); + + @Override + public CompletableFuture createBucketSnapshot(SnapshotMetadata snapshotMetadata, + List bucketSnapshotSegments, + String bucketKey, String topicName, String cursorName) { + long id = idGenerator.getAndIncrement(); + snapshots.put(id, snapshotMetadata); + snapshotSegments.put(id, new ArrayList<>(bucketSnapshotSegments)); + long snapshotLength = snapshotMetadata.toByteArray().length; + for (SnapshotSegment bucketSnapshotSegment : bucketSnapshotSegments) { + snapshotLength += bucketSnapshotSegment.toByteArray().length; + } + snapshotLengths.put(id, snapshotLength); + return CompletableFuture.completedFuture(id); + } + + @Override + public CompletableFuture getBucketSnapshotMetadata(long bucketId) { + SnapshotMetadata metadata = snapshots.get(bucketId); + return CompletableFuture.completedFuture(metadata); + } + + @Override + public CompletableFuture> getBucketSnapshotSegment(long bucketId, + long firstSegmentEntryId, + long lastSegmentEntryId) { + List segments = snapshotSegments.get(bucketId); + if (segments == null) { + return CompletableFuture.failedFuture( + new IllegalArgumentException("Bucket snapshot segments not found: " + bucketId)); + } + if (firstSegmentEntryId > lastSegmentEntryId) { + return CompletableFuture.completedFuture(Collections.emptyList()); + } + + int fromIndex = Math.toIntExact(firstSegmentEntryId - 1); + int toIndex = Math.toIntExact(lastSegmentEntryId); + if (fromIndex < 0 || fromIndex >= segments.size()) { + return CompletableFuture.failedFuture( + new IllegalArgumentException("Invalid first segment entry id: " + firstSegmentEntryId)); + } + toIndex = Math.min(toIndex, segments.size()); + return CompletableFuture.completedFuture(new ArrayList<>(segments.subList(fromIndex, toIndex))); + } + + @Override + public CompletableFuture getBucketSnapshotLength(long bucketId) { + return CompletableFuture.completedFuture(snapshotLengths.getOrDefault(bucketId, 0L)); + } + + @Override + public CompletableFuture deleteBucketSnapshot(long bucketId) { + snapshots.remove(bucketId); + snapshotSegments.remove(bucketId); + snapshotLengths.remove(bucketId); + return CompletableFuture.completedFuture(null); + } + + @Override + public void start() throws Exception { + // No-op + } + + @Override + public void close() throws Exception { + snapshots.clear(); + snapshotSegments.clear(); + snapshotLengths.clear(); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/AbstractDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/AbstractDelayedDeliveryTracker.java index bec5134c4f79a..47753335db8cc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/AbstractDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/AbstractDelayedDeliveryTracker.java @@ -29,7 +29,7 @@ @Slf4j public abstract class AbstractDelayedDeliveryTracker implements DelayedDeliveryTracker, TimerTask { - protected final AbstractPersistentDispatcherMultipleConsumers dispatcher; + protected final DelayedDeliveryContext context; // Reference to the shared (per-broker) timer for delayed delivery protected final Timer timer; @@ -48,24 +48,39 @@ public abstract class AbstractDelayedDeliveryTracker implements DelayedDeliveryT protected final Clock clock; private final boolean isDelayedDeliveryDeliverAtTimeStrict; + private final Object triggerLock; public AbstractDelayedDeliveryTracker(AbstractPersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis, boolean isDelayedDeliveryDeliverAtTimeStrict) { - this(dispatcher, timer, tickTimeMillis, Clock.systemUTC(), isDelayedDeliveryDeliverAtTimeStrict); + this(new DispatcherDelayedDeliveryContext(dispatcher), timer, tickTimeMillis, + Clock.systemUTC(), isDelayedDeliveryDeliverAtTimeStrict); } public AbstractDelayedDeliveryTracker(AbstractPersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis, Clock clock, boolean isDelayedDeliveryDeliverAtTimeStrict) { - this.dispatcher = dispatcher; + this(new DispatcherDelayedDeliveryContext(dispatcher), timer, tickTimeMillis, + clock, isDelayedDeliveryDeliverAtTimeStrict); + } + + public AbstractDelayedDeliveryTracker(DelayedDeliveryContext context, Timer timer, + long tickTimeMillis, + boolean isDelayedDeliveryDeliverAtTimeStrict) { + this(context, timer, tickTimeMillis, Clock.systemUTC(), isDelayedDeliveryDeliverAtTimeStrict); + } + + public AbstractDelayedDeliveryTracker(DelayedDeliveryContext context, Timer timer, + long tickTimeMillis, Clock clock, + boolean isDelayedDeliveryDeliverAtTimeStrict) { + this.context = context; + this.triggerLock = context.getTriggerLock(); this.timer = timer; this.tickTimeMillis = tickTimeMillis; this.clock = clock; this.isDelayedDeliveryDeliverAtTimeStrict = isDelayedDeliveryDeliverAtTimeStrict; } - /** * When {@link #isDelayedDeliveryDeliverAtTimeStrict} is false, we allow for early delivery by as much as the * {@link #tickTimeMillis} because it is a slight optimization to let messages skip going back into the delay @@ -124,7 +139,7 @@ protected void updateTimer() { long calculatedDelayMillis = Math.max(delayMillis, remainingTickDelayMillis); if (log.isDebugEnabled()) { - log.debug("[{}] Start timer in {} millis", dispatcher.getName(), calculatedDelayMillis); + log.debug("[{}] Start timer in {} millis", context.getName(), calculatedDelayMillis); } // Even though we may delay longer than this timestamp because of the tick delay, we still track the @@ -136,17 +151,17 @@ protected void updateTimer() { @Override public void run(Timeout timeout) throws Exception { if (log.isDebugEnabled()) { - log.debug("[{}] Timer triggered", dispatcher.getName()); + log.debug("[{}] Timer triggered", context.getName()); } if (timeout == null || timeout.isCancelled()) { return; } - synchronized (dispatcher) { + synchronized (triggerLock) { lastTickRun = clock.millis(); currentTimeoutTarget = -1; this.timeout = null; - dispatcher.readMoreEntriesAsync(); + context.triggerReadMoreEntries(); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryContext.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryContext.java new file mode 100644 index 0000000000000..a94d5258f1945 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedDeliveryContext.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.delayed; + +import org.apache.bookkeeper.mledger.ManagedCursor; + +public interface DelayedDeliveryContext { + + String getName(); + + ManagedCursor getCursor(); + + Object getTriggerLock(); + + void triggerReadMoreEntries(); +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DispatcherDelayedDeliveryContext.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DispatcherDelayedDeliveryContext.java new file mode 100644 index 0000000000000..2ea388d504c24 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DispatcherDelayedDeliveryContext.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.delayed; + +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.pulsar.broker.service.persistent.AbstractPersistentDispatcherMultipleConsumers; + +public class DispatcherDelayedDeliveryContext implements DelayedDeliveryContext { + + private final AbstractPersistentDispatcherMultipleConsumers dispatcher; + + public DispatcherDelayedDeliveryContext(AbstractPersistentDispatcherMultipleConsumers dispatcher) { + this.dispatcher = dispatcher; + } + + @Override + public String getName() { + return dispatcher.getName(); + } + + @Override + public ManagedCursor getCursor() { + return dispatcher.getCursor(); + } + + @Override + public Object getTriggerLock() { + return dispatcher; + } + + @Override + public void triggerReadMoreEntries() { + dispatcher.readMoreEntriesAsync(); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java index bdc6e4c814e33..e94bb32a6f45c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java @@ -56,15 +56,24 @@ public class InMemoryDelayedDeliveryTracker extends AbstractDelayedDeliveryTrack long tickTimeMillis, boolean isDelayedDeliveryDeliverAtTimeStrict, long fixedDelayDetectionLookahead) { - this(dispatcher, timer, tickTimeMillis, Clock.systemUTC(), isDelayedDeliveryDeliverAtTimeStrict, - fixedDelayDetectionLookahead); + this(new DispatcherDelayedDeliveryContext(dispatcher), timer, tickTimeMillis, Clock.systemUTC(), + isDelayedDeliveryDeliverAtTimeStrict, fixedDelayDetectionLookahead); } + @VisibleForTesting public InMemoryDelayedDeliveryTracker(AbstractPersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis, Clock clock, boolean isDelayedDeliveryDeliverAtTimeStrict, long fixedDelayDetectionLookahead) { - super(dispatcher, timer, tickTimeMillis, clock, isDelayedDeliveryDeliverAtTimeStrict); + this(new DispatcherDelayedDeliveryContext(dispatcher), timer, tickTimeMillis, clock, + isDelayedDeliveryDeliverAtTimeStrict, fixedDelayDetectionLookahead); + } + + private InMemoryDelayedDeliveryTracker(DelayedDeliveryContext context, Timer timer, + long tickTimeMillis, Clock clock, + boolean isDelayedDeliveryDeliverAtTimeStrict, + long fixedDelayDetectionLookahead) { + super(context, timer, tickTimeMillis, clock, isDelayedDeliveryDeliverAtTimeStrict); this.fixedDelayDetectionLookahead = fixedDelayDetectionLookahead; } @@ -76,7 +85,7 @@ public boolean addMessage(long ledgerId, long entryId, long deliverAt) { } if (log.isDebugEnabled()) { - log.debug("[{}] Add message {}:{} -- Delivery in {} ms ", dispatcher.getName(), ledgerId, entryId, + log.debug("[{}] Add message {}:{} -- Delivery in {} ms ", context.getName(), ledgerId, entryId, deliverAt - clock.millis()); } @@ -136,7 +145,7 @@ public NavigableSet getScheduledMessages(int maxMessages) { } if (log.isDebugEnabled()) { - log.debug("[{}] Get scheduled messages - found {}", dispatcher.getName(), positions.size()); + log.debug("[{}] Get scheduled messages - found {}", context.getName(), positions.size()); } if (priorityQueue.isEmpty()) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/NoopDelayedDeliveryContext.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/NoopDelayedDeliveryContext.java new file mode 100644 index 0000000000000..826e127df005a --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/NoopDelayedDeliveryContext.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.delayed; + +import org.apache.bookkeeper.mledger.ManagedCursor; + +public class NoopDelayedDeliveryContext implements DelayedDeliveryContext { + + private final String name; + private final ManagedCursor cursor; + private final Object triggerLock = new Object(); + + public NoopDelayedDeliveryContext(String name, ManagedCursor cursor) { + this.name = name; + this.cursor = cursor; + } + + @Override + public String getName() { + return name; + } + + @Override + public ManagedCursor getCursor() { + return cursor; + } + + @Override + public Object getTriggerLock() { + return triggerLock; + } + + @Override + public void triggerReadMoreEntries() { + // no-op; for tests/JMH + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index 0091f6a0f0266..514dc0900dfab 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -57,6 +57,8 @@ import org.apache.commons.lang3.mutable.MutableLong; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.delayed.AbstractDelayedDeliveryTracker; +import org.apache.pulsar.broker.delayed.DelayedDeliveryContext; +import org.apache.pulsar.broker.delayed.DispatcherDelayedDeliveryContext; import org.apache.pulsar.broker.delayed.proto.DelayedIndex; import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; import org.apache.pulsar.broker.service.persistent.AbstractPersistentDispatcherMultipleConsumers; @@ -123,9 +125,9 @@ public BucketDelayedDeliveryTracker(AbstractPersistentDispatcherMultipleConsumer long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegmentInMillis, int maxIndexesPerBucketSnapshotSegment, int maxNumBuckets) throws RecoverDelayedDeliveryTrackerException { - this(dispatcher, timer, tickTimeMillis, Clock.systemUTC(), isDelayedDeliveryDeliverAtTimeStrict, - bucketSnapshotStorage, minIndexCountPerBucket, timeStepPerBucketSnapshotSegmentInMillis, - maxIndexesPerBucketSnapshotSegment, maxNumBuckets); + this(new DispatcherDelayedDeliveryContext(dispatcher), timer, tickTimeMillis, Clock.systemUTC(), + isDelayedDeliveryDeliverAtTimeStrict, bucketSnapshotStorage, minIndexCountPerBucket, + timeStepPerBucketSnapshotSegmentInMillis, maxIndexesPerBucketSnapshotSegment, maxNumBuckets); } public BucketDelayedDeliveryTracker(AbstractPersistentDispatcherMultipleConsumers dispatcher, @@ -135,7 +137,20 @@ public BucketDelayedDeliveryTracker(AbstractPersistentDispatcherMultipleConsumer long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegmentInMillis, int maxIndexesPerBucketSnapshotSegment, int maxNumBuckets) throws RecoverDelayedDeliveryTrackerException { - super(dispatcher, timer, tickTimeMillis, clock, isDelayedDeliveryDeliverAtTimeStrict); + this(new DispatcherDelayedDeliveryContext(dispatcher), timer, tickTimeMillis, clock, + isDelayedDeliveryDeliverAtTimeStrict, bucketSnapshotStorage, minIndexCountPerBucket, + timeStepPerBucketSnapshotSegmentInMillis, maxIndexesPerBucketSnapshotSegment, maxNumBuckets); + } + + @VisibleForTesting + public BucketDelayedDeliveryTracker(DelayedDeliveryContext context, + Timer timer, long tickTimeMillis, Clock clock, + boolean isDelayedDeliveryDeliverAtTimeStrict, + BucketSnapshotStorage bucketSnapshotStorage, + long minIndexCountPerBucket, long timeStepPerBucketSnapshotSegmentInMillis, + int maxIndexesPerBucketSnapshotSegment, int maxNumBuckets) + throws RecoverDelayedDeliveryTrackerException { + super(context, timer, tickTimeMillis, clock, isDelayedDeliveryDeliverAtTimeStrict); this.minIndexCountPerBucket = minIndexCountPerBucket; this.timeStepPerBucketSnapshotSegmentInMillis = timeStepPerBucketSnapshotSegmentInMillis; this.maxIndexesPerBucketSnapshotSegment = maxIndexesPerBucketSnapshotSegment; @@ -144,7 +159,7 @@ public BucketDelayedDeliveryTracker(AbstractPersistentDispatcherMultipleConsumer this.immutableBuckets = TreeRangeMap.create(); this.snapshotSegmentLastIndexMap = new ConcurrentHashMap<>(); this.lastMutableBucket = - new MutableBucket(dispatcher.getName(), dispatcher.getCursor(), FutureUtil.Sequencer.create(), + new MutableBucket(context.getName(), context.getCursor(), FutureUtil.Sequencer.create(), bucketSnapshotStorage); this.stats = new BucketDelayedMessageIndexStats(); @@ -163,7 +178,7 @@ private synchronized long recoverBucketSnapshot() throws RecoverDelayedDeliveryT Map cursorProperties = cursor.getCursorProperties(); if (MapUtils.isEmpty(cursorProperties)) { log.info("[{}] Recover delayed message index bucket snapshot finish, don't find bucket snapshot", - dispatcher.getName()); + context.getName()); return 0; } FutureUtil.Sequencer sequencer = this.lastMutableBucket.getSequencer(); @@ -173,7 +188,7 @@ private synchronized long recoverBucketSnapshot() throws RecoverDelayedDeliveryT String[] keys = key.split(DELIMITER); checkArgument(keys.length == 3); ImmutableBucket immutableBucket = - new ImmutableBucket(dispatcher.getName(), cursor, sequencer, + new ImmutableBucket(context.getName(), cursor, sequencer, this.lastMutableBucket.bucketSnapshotStorage, Long.parseLong(keys[1]), Long.parseLong(keys[2])); putAndCleanOverlapRange(Range.closed(immutableBucket.startLedgerId, immutableBucket.endLedgerId), @@ -184,7 +199,7 @@ private synchronized long recoverBucketSnapshot() throws RecoverDelayedDeliveryT Map, ImmutableBucket> immutableBucketMap = immutableBuckets.asMapOfRanges(); if (immutableBucketMap.isEmpty()) { log.info("[{}] Recover delayed message index bucket snapshot finish, don't find bucket snapshot", - dispatcher.getName()); + context.getName()); return 0; } @@ -198,7 +213,7 @@ private synchronized long recoverBucketSnapshot() throws RecoverDelayedDeliveryT try { FutureUtil.waitForAll(futures.values()).get(AsyncOperationTimeoutSeconds * 5, TimeUnit.SECONDS); } catch (InterruptedException | ExecutionException | TimeoutException e) { - log.error("[{}] Failed to recover delayed message index bucket snapshot.", dispatcher.getName(), e); + log.error("[{}] Failed to recover delayed message index bucket snapshot.", context.getName(), e); if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); } @@ -239,7 +254,7 @@ private synchronized long recoverBucketSnapshot() throws RecoverDelayedDeliveryT }); log.info("[{}] Recover delayed message index bucket snapshot finish, buckets: {}, numberDelayedMessages: {}", - dispatcher.getName(), immutableBucketMap.size(), numberDelayedMessages.getValue()); + context.getName(), immutableBucketMap.size(), numberDelayedMessages.longValue()); return numberDelayedMessages.getValue(); } @@ -327,7 +342,7 @@ private void afterCreateImmutableBucket(Pair immu if (ex == null) { immutableBucket.setSnapshotSegments(null); immutableBucket.asyncUpdateSnapshotLength(); - log.info("[{}] Create bucket snapshot finish, bucketKey: {}", dispatcher.getName(), + log.info("[{}] Create bucket snapshot finish, bucketKey: {}", context.getName(), immutableBucket.bucketKey()); stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.create, @@ -336,7 +351,7 @@ private void afterCreateImmutableBucket(Pair immu return bucketId; } - log.error("[{}] Failed to create bucket snapshot, bucketKey: {}", dispatcher.getName(), + log.error("[{}] Failed to create bucket snapshot, bucketKey: {}", context.getName(), immutableBucket.bucketKey(), ex); stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.create); @@ -415,7 +430,7 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver numberDelayedMessages.incrementAndGet(); if (log.isDebugEnabled()) { - log.debug("[{}] Add message {}:{} -- Delivery in {} ms ", dispatcher.getName(), ledgerId, entryId, + log.debug("[{}] Add message {}:{} -- Delivery in {} ms ", context.getName(), ledgerId, entryId, deliverAt - clock.millis()); } @@ -477,14 +492,14 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot() { List toBeMergeImmutableBuckets = selectMergedBuckets(immutableBucketList, MAX_MERGE_NUM); if (toBeMergeImmutableBuckets.isEmpty()) { - log.warn("[{}] Can't find able merged buckets", dispatcher.getName()); + log.warn("[{}] Can't find able merged buckets", context.getName()); return CompletableFuture.completedFuture(null); } final String bucketsStr = toBeMergeImmutableBuckets.stream().map(Bucket::bucketKey).collect( Collectors.joining(",")).replaceAll(DELAYED_BUCKET_KEY_PREFIX + "_", ""); if (log.isDebugEnabled()) { - log.info("[{}] Merging bucket snapshot, bucketKeys: {}", dispatcher.getName(), bucketsStr); + log.info("[{}] Merging bucket snapshot, bucketKeys: {}", context.getName(), bucketsStr); } for (ImmutableBucket immutableBucket : toBeMergeImmutableBuckets) { @@ -501,12 +516,12 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot() { } if (ex != null) { log.error("[{}] Failed to merge bucket snapshot, bucketKeys: {}", - dispatcher.getName(), bucketsStr, ex); + context.getName(), bucketsStr, ex); stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.merge); } else { log.info("[{}] Merge bucket snapshot finish, bucketKeys: {}, bucketNum: {}", - dispatcher.getName(), bucketsStr, immutableBuckets.asMapOfRanges().size()); + context.getName(), bucketsStr, immutableBuckets.asMapOfRanges().size()); stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.merge, System.currentTimeMillis() - mergeStartTime); @@ -626,7 +641,7 @@ public synchronized NavigableSet getScheduledMessages(int maxMessages) if (!checkPendingLoadDone()) { if (log.isDebugEnabled()) { log.debug("[{}] Skip getScheduledMessages to wait for bucket snapshot load finish.", - dispatcher.getName()); + context.getName()); } return Collections.emptyNavigableSet(); } @@ -661,19 +676,19 @@ public synchronized NavigableSet getScheduledMessages(int maxMessages) // All message of current snapshot segment are scheduled, try load next snapshot segment if (bucket.merging) { log.info("[{}] Skip load to wait for bucket snapshot merge finish, bucketKey:{}", - dispatcher.getName(), bucket.bucketKey()); + context.getName(), bucket.bucketKey()); break; } final int preSegmentEntryId = bucket.currentSegmentEntryId; if (log.isDebugEnabled()) { log.debug("[{}] Loading next bucket snapshot segment, bucketKey: {}, nextSegmentEntryId: {}", - dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1); + context.getName(), bucket.bucketKey(), preSegmentEntryId + 1); } boolean createFutureDone = bucket.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).isDone(); if (!createFutureDone) { log.info("[{}] Skip load to wait for bucket snapshot create finish, bucketKey:{}", - dispatcher.getName(), bucket.bucketKey()); + context.getName(), bucket.bucketKey()); break; } @@ -705,12 +720,12 @@ public synchronized NavigableSet getScheduledMessages(int maxMessages) bucket.setCurrentSegmentEntryId(preSegmentEntryId); log.error("[{}] Failed to load bucket snapshot segment, bucketKey: {}, segmentEntryId: {}", - dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1, ex); + context.getName(), bucket.bucketKey(), preSegmentEntryId + 1, ex); stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.load); } else { log.info("[{}] Load next bucket snapshot segment finish, bucketKey: {}, segmentEntryId: {}", - dispatcher.getName(), bucket.bucketKey(), + context.getName(), bucket.bucketKey(), (preSegmentEntryId == bucket.lastSegmentEntryId) ? "-1" : preSegmentEntryId + 1); stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.load, @@ -798,7 +813,7 @@ public CompletableFuture closeAsync() { } return FutureUtil.waitForAll(completableFutures) .exceptionally(e -> { - log.warn("[{}] Failed wait to snapshot generate", dispatcher.getName(), e); + log.warn("[{}] Failed wait to snapshot generate", context.getName(), e); return null; }); } @@ -855,7 +870,7 @@ private synchronized CompletableFuture asyncTrimImmutableBuckets() { if (null == firstLedgerId) { return CompletableFuture.completedFuture(null); } - ManagedLedger ledger = dispatcher.getCursor().getManagedLedger(); + ManagedLedger ledger = context.getCursor().getManagedLedger(); Map, ImmutableBucket> toBeDeletedBuckets = new HashMap<>(); // subRangeMap returns clipped intersection ranges. Snapshot deletion must use the original @@ -898,7 +913,7 @@ private CompletableFuture deleteBucketSnapshot(String ledgerName, } private Long firstActiveLedgerId() { - ManagedCursor cursor = dispatcher.getCursor(); + ManagedCursor cursor = context.getCursor(); Position mdp = cursor.getMarkDeletedPosition(); return mdp == null ? null : mdp.getLedgerId(); } From 8f1dc74477e90ea690a94566faed94e903ed6ab7 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Mon, 15 Jun 2026 18:46:38 +0800 Subject: [PATCH 192/213] [improve][broker] Optimize TripleLongPriorityQueue heap operations (#26010) --- .../TripleLongPriorityQueueBenchmark.java | 168 ++++++++++++++++++ .../collections/TripleLongPriorityQueue.java | 156 ++++++++++------ .../TripleLongPriorityQueueTest.java | 47 +++++ 3 files changed, 315 insertions(+), 56 deletions(-) create mode 100644 microbench/src/main/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueueBenchmark.java diff --git a/microbench/src/main/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueueBenchmark.java b/microbench/src/main/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueueBenchmark.java new file mode 100644 index 0000000000000..1e50ab600bfae --- /dev/null +++ b/microbench/src/main/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueueBenchmark.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.common.util.collections; + +import java.util.Random; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * JMH benchmarks for {@link TripleLongPriorityQueue} simulating Pulsar delayed delivery workloads. + * + *

Three scenarios matching real usage: + *

    + *
  • {@link #recoveryBulkAddThenPop} — snapshot recovery: bulk add all entries, then pop all. + * Cold cache, large heap. This is the worst case for the hole-based optimization + * because cache misses dominate over reduced readLong calls.
    • + *
  • {@link #interleavedAddPop} — steady-state delayed delivery: batch add (messages arriving + * between timer ticks), then batch pop (getScheduledMessages). Heap stays warm in cache. + * This is the primary hot path.
    • + *
  • {@link #steadyState} — constant-depth steady state: pre-fill queue, then alternating + * small add/pop batches. Simulates sustained throughput with ~10K queue depth.
    • + *
+ * + *

Build and run: + * 

+ * ./gradlew :microbench:shadowJar
+ * java -jar microbench/build/libs/microbench-*-benchmarks.jar ".*TripleLongPriorityQueue.*"
+ *
+ */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Fork(2) +@Warmup(iterations = 3, time = 1) +@Measurement(iterations = 5, time = 1) +@State(Scope.Thread) +public class TripleLongPriorityQueueBenchmark { + + @Param({"50000", "500000", "2000000"}) + int size; + + /** + * Recovery scenario: bulk add all then pop all. + * Simulates snapshot recovery from BookKeeper — cold cache, large heap. + */ + @Benchmark + public void recoveryBulkAddThenPop(Blackhole bh) { + try (TripleLongPriorityQueue pq = new TripleLongPriorityQueue()) { + long baseTs = System.currentTimeMillis(); + Random rng = new Random(42); + for (int i = 0; i < size; i++) { + long n1 = baseTs + rng.nextLong(3_600_000); + long n2 = i / 1000; + long n3 = i; + pq.add(n1, n2, n3); + } + while (!pq.isEmpty()) { + bh.consume(pq.peekN1()); + pq.pop(); + } + } + } + + /** + * Interleaved scenario: batch add then batch pop, repeated. + * Simulates steady-state delayed delivery — messages arrive continuously, + * getScheduledMessages pops in batches of ~500 when consumers are ready. + * Heap is warm in L2/L3 cache between operations. + */ + @Benchmark + public void interleavedAddPop(Blackhole bh) { + try (TripleLongPriorityQueue pq = new TripleLongPriorityQueue()) { + Random rng = new Random(42); + long baseTs = System.currentTimeMillis(); + int batchSize = 500; + int totalAdded = 0; + + while (totalAdded < size) { + // Batch add: messages arriving between timer ticks + int addCount = Math.min(batchSize + rng.nextInt(500), size - totalAdded); + for (int i = 0; i < addCount; i++) { + long n1 = baseTs + rng.nextLong(3_600_000); + long n2 = (totalAdded + i) / 1000; + long n3 = totalAdded + i; + pq.add(n1, n2, n3); + } + totalAdded += addCount; + + // Batch pop: getScheduledMessages delivering to consumers + int popCount = (int) Math.min(batchSize, pq.size()); + for (int i = 0; i < popCount; i++) { + bh.consume(pq.peekN1()); + pq.pop(); + } + } + // Drain remaining + while (!pq.isEmpty()) { + bh.consume(pq.peekN1()); + pq.pop(); + } + } + } + + /** + * Steady-state scenario: pre-fill queue, then alternating small add/pop. + * Simulates sustained throughput with constant ~10K queue depth. + * Heap stays hot in cache — this is where the readLong reduction matters most. + */ + @Benchmark + public void steadyState(Blackhole bh) { + int steadyDepth = 10_000; + try (TripleLongPriorityQueue pq = new TripleLongPriorityQueue()) { + Random rng = new Random(42); + long baseTs = System.currentTimeMillis(); + long seq = 0; + + // Pre-fill + for (int i = 0; i < steadyDepth; i++) { + pq.add(baseTs + rng.nextLong(3_600_000), seq / 1000, seq); + seq++; + } + + // Alternating add/pop to maintain steady depth + int ops = 0; + while (ops < size) { + // Small batch add + int addCount = 100 + rng.nextInt(100); + for (int i = 0; i < addCount && ops < size; i++) { + pq.add(baseTs + rng.nextLong(3_600_000), seq / 1000, seq); + seq++; + ops++; + } + // Small batch pop + int popCount = addCount - rng.nextInt(20); + for (int i = 0; i < popCount && !pq.isEmpty(); i++) { + bh.consume(pq.peekN1()); + pq.pop(); + } + } + } + } +} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueue.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueue.java index e75ae21ab8518..cd878c6428459 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueue.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueue.java @@ -24,6 +24,25 @@ * Provides a priority-queue implementation specialized on items composed by 3 longs. * *

This class is not thread safe and the items are stored in direct memory. + * + *

Algorithm

+ * + *

This is a binary min-heap stored in a flat array, where each heap node occupies + * 3 consecutive longs (the tuple). The children of the node at index {@code i} are at + * {@code 2i + 1} and {@code 2i + 2}; the parent of node {@code i} is at {@code (i - 1) / 2}. + * + *

Both {@code siftUp} (on insert) and {@code siftDown} (on remove) use the + * hole-based (also called "bottom-up" or "Floyd's") optimization: instead of swapping + * the displaced element with its parent/child at each level, the displaced values are held in + * local variables (registers) and written only once at the final position. This reduces the + * number of array writes per sift layer from 6 (swap: 3 reads + 3 writes on each side) to 3 + * (one directional write), and avoids re-reading the displaced element from the array on every + * comparison. + * + *

Comparison is lexicographic on (n1, n2, n3), using {@code Long.compare} at each level. + * + * @see Bottom-up heapsort + * (Wikipedia) */ public class TripleLongPriorityQueue implements AutoCloseable { private static final int DEFAULT_INITIAL_CAPACITY = 16; @@ -94,8 +113,7 @@ public void add(long n1, long n2, long n3) { array.increaseCapacity(); } - put(tuplesCount, n1, n2, n3); - siftUp(tuplesCount); + siftUp(tuplesCount, n1, n2, n3); ++tuplesCount; } @@ -134,9 +152,17 @@ public long peekN3() { */ public void pop() { checkArgument(tuplesCount != 0); - swap(0, tuplesCount - 1); - tuplesCount--; - siftDown(0); + + if (--tuplesCount == 0) { + return; + } + + long lastBase = tuplesCount * ITEMS_COUNT; + long n1 = array.readLong(lastBase); + long n2 = array.readLong(lastBase + 1); + long n3 = array.readLong(lastBase + 2); + + siftDown(0, n1, n2, n3); shrinkCapacity(); } @@ -188,81 +214,99 @@ private void shrinkCapacity() { } } - private void siftUp(long tupleIdx) { + private void siftUp(long tupleIdx, long n1, long n2, long n3) { + long idx = tupleIdx * ITEMS_COUNT; + while (tupleIdx > 0) { - long parentIdx = (tupleIdx - 1) / 2; - if (compare(tupleIdx, parentIdx) >= 0) { + long parentIdx = (tupleIdx - 1) >>> 1; + long parentBase = parentIdx * ITEMS_COUNT; + + long p0 = array.readLong(parentBase); + long p1 = array.readLong(parentBase + 1); + long p2 = array.readLong(parentBase + 2); + + if (compareTuple(n1, n2, n3, p0, p1, p2) >= 0) { break; } - swap(tupleIdx, parentIdx); + array.writeLong(idx, p0); + array.writeLong(idx + 1, p1); + array.writeLong(idx + 2, p2); + tupleIdx = parentIdx; + idx = parentBase; } + + array.writeLong(idx, n1); + array.writeLong(idx + 1, n2); + array.writeLong(idx + 2, n3); } - private void siftDown(long tupleIdx) { - long half = tuplesCount / 2; + private void siftDown(long tupleIdx, long val0, long val1, long val2) { + long half = tuplesCount >>> 1; + + long idx = tupleIdx * ITEMS_COUNT; + while (tupleIdx < half) { - long left = 2 * tupleIdx + 1; - long right = 2 * tupleIdx + 2; + long left = (tupleIdx << 1) + 1; + long right = left + 1; - long swapIdx = tupleIdx; + long child = left; + long childBase = left * ITEMS_COUNT; - if (compare(tupleIdx, left) > 0) { - swapIdx = left; - } + long child0 = array.readLong(childBase); + long child1 = array.readLong(childBase + 1); + long child2 = array.readLong(childBase + 2); - if (right < tuplesCount && compare(swapIdx, right) > 0) { - swapIdx = right; - } + if (right < tuplesCount) { + long rightBase = right * ITEMS_COUNT; - if (swapIdx == tupleIdx) { - return; - } + long right0 = array.readLong(rightBase); + long right1 = array.readLong(rightBase + 1); + long right2 = array.readLong(rightBase + 2); - swap(tupleIdx, swapIdx); - tupleIdx = swapIdx; - } - } + if (compareTuple(right0, right1, right2, child0, child1, child2) < 0) { - private void put(long tupleIdx, long n1, long n2, long n3) { - long idx = tupleIdx * ITEMS_COUNT; - array.writeLong(idx, n1); - array.writeLong(idx + 1, n2); - array.writeLong(idx + 2, n3); - } + child = right; + childBase = rightBase; - private int compare(long tupleIdx1, long tupleIdx2) { - long idx1 = tupleIdx1 * ITEMS_COUNT; - long idx2 = tupleIdx2 * ITEMS_COUNT; + child0 = right0; + child1 = right1; + child2 = right2; + } + } - int c1 = Long.compare(array.readLong(idx1), array.readLong(idx2)); - if (c1 != 0) { - return c1; - } + if (compareTuple(val0, val1, val2, child0, child1, child2) <= 0) { + break; + } + + array.writeLong(idx, child0); + array.writeLong(idx + 1, child1); + array.writeLong(idx + 2, child2); - int c2 = Long.compare(array.readLong(idx1 + 1), array.readLong(idx2 + 1)); - if (c2 != 0) { - return c2; + tupleIdx = child; + idx = childBase; } - return Long.compare(array.readLong(idx1 + 2), array.readLong(idx2 + 2)); + array.writeLong(idx, val0); + array.writeLong(idx + 1, val1); + array.writeLong(idx + 2, val2); } - private void swap(long tupleIdx1, long tupleIdx2) { - long idx1 = tupleIdx1 * ITEMS_COUNT; - long idx2 = tupleIdx2 * ITEMS_COUNT; + private static int compareTuple( + long a0, long a1, long a2, + long b0, long b1, long b2) { - long tmp1 = array.readLong(idx1); - long tmp2 = array.readLong(idx1 + 1); - long tmp3 = array.readLong(idx1 + 2); + int c = Long.compare(a0, b0); + if (c != 0) { + return c; + } - array.writeLong(idx1, array.readLong(idx2)); - array.writeLong(idx1 + 1, array.readLong(idx2 + 1)); - array.writeLong(idx1 + 2, array.readLong(idx2 + 2)); + c = Long.compare(a1, b1); + if (c != 0) { + return c; + } - array.writeLong(idx2, tmp1); - array.writeLong(idx2 + 1, tmp2); - array.writeLong(idx2 + 2, tmp3); + return Long.compare(a2, b2); } } diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueueTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueueTest.java index d3fcc192b573e..ee5c671d456cd 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueueTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueueTest.java @@ -20,8 +20,12 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import java.util.Comparator; +import java.util.PriorityQueue; +import java.util.Random; import org.testng.annotations.Test; public class TripleLongPriorityQueueTest { @@ -196,4 +200,47 @@ private void triggerScaleOut(int initialCapacity, TripleLongPriorityQueue pq) { pq.add(i, i, i); } } + + @Test + public void testDifferentialRandomPriorityQueue() { + Comparator cmp = Comparator.comparingLong((long[] t) -> t[0]) + .thenComparingLong(t -> t[1]) + .thenComparingLong(t -> t[2]); + + for (int trial = 0; trial < 10; trial++) { + Random rng = new Random(42 + trial); + PriorityQueue oracle = new PriorityQueue<>(cmp); + try (TripleLongPriorityQueue pq = new TripleLongPriorityQueue()) { + int ops = 10_000 + rng.nextInt(10_000); + for (int i = 0; i < ops; i++) { + boolean doAdd = oracle.isEmpty() || rng.nextBoolean(); + if (doAdd) { + // ~10% chance of same-prefix (small n1 range) to exercise tie-breaking + long n1 = rng.nextInt(100) < 10 ? rng.nextLong(20) : rng.nextLong(1_000_000); + long n2 = rng.nextLong(100); + long n3 = rng.nextLong(1_000_000); + oracle.add(new long[]{n1, n2, n3}); + pq.add(n1, n2, n3); + } else { + long[] expected = oracle.poll(); + assertNotNull(expected); + assertEquals(pq.peekN1(), expected[0], "n1 mismatch at op " + i); + assertEquals(pq.peekN2(), expected[1], "n2 mismatch at op " + i); + assertEquals(pq.peekN3(), expected[2], "n3 mismatch at op " + i); + pq.pop(); + } + assertEquals(pq.size(), oracle.size(), "size mismatch at op " + i); + } + // drain remaining + while (!oracle.isEmpty()) { + long[] expected = oracle.poll(); + assertEquals(pq.peekN1(), expected[0]); + assertEquals(pq.peekN2(), expected[1]); + assertEquals(pq.peekN3(), expected[2]); + pq.pop(); + } + assertTrue(pq.isEmpty()); + } + } + } } From 5e8dcf80dd50a190076da24aa59c41ed6cac1f54 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Wed, 1 Jul 2026 09:43:59 +0800 Subject: [PATCH 193/213] [improve][broker] Add LongBitmap abstraction and migrate RoaringBitmap usage to LongBitmap (#26117) --- .../util/collections/LongBitmapBenchmark.java | 141 +++ .../pulsar/broker/delayed/bucket/Bucket.java | 11 +- .../bucket/BucketDelayedDeliveryTracker.java | 6 +- .../delayed/bucket/ImmutableBucket.java | 36 +- .../broker/delayed/bucket/MutableBucket.java | 23 +- .../service/ConsumerNameIndexTracker.java | 11 +- .../broker/service/DrainingHashesTracker.java | 27 +- pulsar-common/pom.xml | 5 + .../collections/ConcurrentRoaringBitmap.java | 441 ++++++++++ .../common/util/collections/LongBitmap.java | 170 ++++ .../common/util/collections/LongBitmaps.java | 61 ++ .../LongBitmapCompatibilityTest.java | 235 +++++ .../util/collections/LongBitmapTest.java | 824 ++++++++++++++++++ 13 files changed, 1927 insertions(+), 64 deletions(-) create mode 100644 microbench/src/main/java/org/apache/pulsar/common/util/collections/LongBitmapBenchmark.java create mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java create mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongBitmap.java create mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongBitmaps.java create mode 100644 pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/LongBitmapCompatibilityTest.java create mode 100644 pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/LongBitmapTest.java diff --git a/microbench/src/main/java/org/apache/pulsar/common/util/collections/LongBitmapBenchmark.java b/microbench/src/main/java/org/apache/pulsar/common/util/collections/LongBitmapBenchmark.java new file mode 100644 index 0000000000000..9fa3c464d7324 --- /dev/null +++ b/microbench/src/main/java/org/apache/pulsar/common/util/collections/LongBitmapBenchmark.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util.collections; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.roaringbitmap.RoaringBitmap; +import org.roaringbitmap.longlong.Roaring64Bitmap; + +/** + * JMH benchmark for {@link LongBitmap} ({@link ConcurrentRoaringBitmap}), compared against + * the pre-PR bitmap implementations it replaces: + *

    + *
  • {@link Roaring64Bitmap} — previously used in {@code InMemoryDelayedDeliveryTracker}.
    • + *
  • {@link RoaringBitmap} — previously used in {@code ConsumerNameIndexTracker}, + * {@code DrainingHashesTracker}, and the bucket delayed-delivery family.
    • + *
+ * + *

Run with: + * 

{@code
+ * ./gradlew :microbench:shadowJar
+ * java -jar microbench/build/libs/microbench-*-benchmarks.jar LongBitmapBenchmark
+ * }
+ */ +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@State(Scope.Benchmark) +@Warmup(time = 2, iterations = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(time = 3, iterations = 1, timeUnit = TimeUnit.SECONDS) +@Fork(1) +public class LongBitmapBenchmark { + + @Param({"1000", "100000"}) + public int bitmapSize; + + private LongBitmap longBitmap; + private RoaringBitmap roaringBitmap; + private Roaring64Bitmap roaring64Bitmap; + + private final AtomicLong nextValue = new AtomicLong(); + + @Setup(Level.Trial) + public void setup() { + longBitmap = LongBitmaps.create(); + roaringBitmap = new RoaringBitmap(); + roaring64Bitmap = new Roaring64Bitmap(); + for (int i = 0; i < bitmapSize; i++) { + longBitmap.add(i); + roaringBitmap.add(i); + roaring64Bitmap.addLong(i); + } + nextValue.set(bitmapSize); + } + + @Benchmark + @Threads(1) + public boolean longBitmapAddSingleThread() { + long v = nextValue.getAndIncrement(); + return longBitmap.checkedAdd(v); + } + + @Benchmark + @Threads(1) + public boolean roaringBitmapAddSingleThread() { + long v = nextValue.getAndIncrement(); + return roaringBitmap.checkedAdd((int) v); + } + + @Benchmark + @Threads(1) + public boolean roaring64BitmapAddSingleThread() { + long v = nextValue.getAndIncrement(); + boolean existed = roaring64Bitmap.contains(v); + roaring64Bitmap.addLong(v); + return !existed; + } + + @Benchmark + @Threads(1) + public boolean longBitmapContainsSingleThread() { + return longBitmap.contains(nextValue.getAndIncrement() % bitmapSize); + } + + @Benchmark + @Threads(1) + public boolean roaringBitmapContainsSingleThread() { + return roaringBitmap.contains((int) (nextValue.getAndIncrement() % bitmapSize)); + } + + @Benchmark + @Threads(1) + public boolean roaring64BitmapContainsSingleThread() { + return roaring64Bitmap.contains(nextValue.getAndIncrement() % bitmapSize); + } + + // Bare RoaringBitmap variants are not thread-safe and are omitted below. + + @Benchmark + @Threads(4) + public boolean longBitmapAdd4Threads() { + long v = nextValue.getAndIncrement(); + return longBitmap.checkedAdd(v); + } + + @Benchmark + @Threads(4) + public void longBitmapContains4Threads(Blackhole bh) { + int v = (int) (Thread.currentThread().getId() * 31 + System.nanoTime()); + bh.consume(longBitmap.contains(Math.abs(v) % bitmapSize)); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java index 776f99b120c3d..d27ca5782449d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java @@ -34,7 +34,8 @@ import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.FutureUtil; -import org.roaringbitmap.RoaringBitmap; +import org.apache.pulsar.common.util.collections.LongBitmap; +import org.apache.pulsar.common.util.collections.LongBitmaps; @Slf4j @Data @@ -55,7 +56,7 @@ abstract class Bucket { long startLedgerId; long endLedgerId; - Map delayedIndexBitMap; + Map delayedIndexBitMap; long numberBucketDelayedMessages; @@ -77,7 +78,7 @@ abstract class Bucket { } boolean containsMessage(long ledgerId, long entryId) { - RoaringBitmap bitSet = delayedIndexBitMap.get(ledgerId); + LongBitmap bitSet = delayedIndexBitMap.get(ledgerId); if (bitSet == null) { return false; } @@ -85,12 +86,12 @@ boolean containsMessage(long ledgerId, long entryId) { } void putIndexBit(long ledgerId, long entryId) { - delayedIndexBitMap.computeIfAbsent(ledgerId, k -> new RoaringBitmap()).add(entryId, entryId + 1); + delayedIndexBitMap.computeIfAbsent(ledgerId, k -> LongBitmaps.create()).add(entryId, entryId + 1); } boolean removeIndexBit(long ledgerId, long entryId) { boolean contained = false; - RoaringBitmap bitSet = delayedIndexBitMap.get(ledgerId); + LongBitmap bitSet = delayedIndexBitMap.get(ledgerId); if (bitSet != null && bitSet.contains(entryId, entryId + 1)) { contained = true; bitSet.remove(entryId, entryId + 1); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index 514dc0900dfab..3a889ac223524 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -64,8 +64,8 @@ import org.apache.pulsar.broker.service.persistent.AbstractPersistentDispatcherMultipleConsumers; import org.apache.pulsar.common.policies.data.stats.TopicMetricBean; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.collections.LongBitmap; import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue; -import org.roaringbitmap.RoaringBitmap; @Slf4j @ThreadSafe @@ -560,7 +560,7 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(List delayedIndexBitMap = + Map delayedIndexBitMap = new HashMap<>(buckets.get(0).getDelayedIndexBitMap()); for (int i = 1; i < buckets.size(); i++) { buckets.get(i).delayedIndexBitMap.forEach((ledgerId, bitMapB) -> { @@ -575,8 +575,6 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(List> asyncLoadNextBucketSnapshotEntry(b /** * Recover delayed index bit map and message numbers. - * @throws InvalidRoaringFormat invalid bitmap serialization format */ private void recoverDelayedIndexBitMapAndNumber(int startSnapshotIndex, List segmentMetaList) { @@ -149,25 +149,23 @@ private void recoverDelayedIndexBitMapAndNumber(int startSnapshotIndex, for (final var entry : segmentMetaList.get(i).getDelayedIndexBitMapMap().entrySet()) { final var ledgerId = entry.getKey(); final var bs = entry.getValue(); - final var sbm = new RoaringBitmap(); + final ByteBuf buf = Unpooled.wrappedBuffer(bs.asReadOnlyByteBuffer()); try { - sbm.deserialize(bs.asReadOnlyByteBuffer()); - } catch (IOException e) { - throw new InvalidRoaringFormat(e.getMessage()); + final LongBitmap sbm = LongBitmaps.deserialize(buf); + numberMessages.add(sbm.cardinality()); + delayedIndexBitMap.compute(ledgerId, (lId, bm) -> { + if (bm == null) { + return sbm; + } + bm.or(sbm); + return bm; + }); + } finally { + buf.release(); } - numberMessages.add(sbm.getCardinality()); - delayedIndexBitMap.compute(ledgerId, (lId, bm) -> { - if (bm == null) { - return sbm; - } - bm.or(sbm); - return bm; - }); } } - // optimize bm - delayedIndexBitMap.values().forEach(RoaringBitmap::runOptimize); - setNumberBucketDelayedMessages(numberMessages.getValue()); + setNumberBucketDelayedMessages(numberMessages.longValue()); } CompletableFuture> getRemainSnapshotSegment() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java index 1173a401a8903..ffdb652a3820d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java @@ -20,7 +20,6 @@ import static com.google.common.base.Preconditions.checkArgument; import com.google.protobuf.UnsafeByteOperations; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; @@ -35,8 +34,9 @@ import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; import org.apache.pulsar.broker.delayed.proto.SnapshotSegmentMetadata; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.collections.LongBitmap; +import org.apache.pulsar.common.util.collections.LongBitmaps; import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue; -import org.roaringbitmap.RoaringBitmap; @Slf4j class MutableBucket extends Bucket implements AutoCloseable { @@ -74,9 +74,9 @@ Pair createImmutableBucketAndAsyncPersistent( List bucketSnapshotSegments = new ArrayList<>(); List segmentMetadataList = new ArrayList<>(); - Map immutableBucketBitMap = new HashMap<>(); + Map immutableBucketBitMap = new HashMap<>(); - Map bitMap = new HashMap<>(); + Map bitMap = new HashMap<>(); SnapshotSegment snapshotSegment = new SnapshotSegment(); SnapshotSegmentMetadata.Builder segmentMetadataBuilder = SnapshotSegmentMetadata.newBuilder(); @@ -106,7 +106,7 @@ Pair createImmutableBucketAndAsyncPersistent( sharedQueue.add(timestamp, ledgerId, entryId); } - bitMap.computeIfAbsent(ledgerId, k -> new RoaringBitmap()).add(entryId, entryId + 1); + bitMap.computeIfAbsent(ledgerId, k -> LongBitmaps.create()).add(entryId, entryId + 1); numMessages++; @@ -117,16 +117,13 @@ Pair createImmutableBucketAndAsyncPersistent( segmentMetadataBuilder.setMinScheduleTimestamp(currentFirstTimestamp); currentTimestampUpperLimit = 0; - Iterator> iterator = bitMap.entrySet().iterator(); + Iterator> iterator = bitMap.entrySet().iterator(); while (iterator.hasNext()) { final var entry = iterator.next(); final var lId = entry.getKey(); final var bm = entry.getValue(); - bm.runOptimize(); - ByteBuffer byteBuffer = ByteBuffer.allocate(bm.serializedSizeInBytes()); - bm.serialize(byteBuffer); - byteBuffer.flip(); - segmentMetadataBuilder.putDelayedIndexBitMap(lId, UnsafeByteOperations.unsafeWrap(byteBuffer)); + segmentMetadataBuilder.putDelayedIndexBitMap(lId, + UnsafeByteOperations.unsafeWrap(bm.serialize())); immutableBucketBitMap.compute(lId, (__, bm0) -> { if (bm0 == null) { return bm; @@ -145,10 +142,6 @@ Pair createImmutableBucketAndAsyncPersistent( } } - // optimize bm - immutableBucketBitMap.values().forEach(RoaringBitmap::runOptimize); - this.delayedIndexBitMap.values().forEach(RoaringBitmap::runOptimize); - SnapshotMetadata bucketSnapshotMetadata = SnapshotMetadata.newBuilder() .addAllMetadataList(segmentMetadataList) .build(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsumerNameIndexTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsumerNameIndexTracker.java index 1f93313ab1b71..3ab9e013c2743 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsumerNameIndexTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsumerNameIndexTracker.java @@ -22,7 +22,8 @@ import java.util.Map; import javax.annotation.concurrent.NotThreadSafe; import org.apache.commons.lang3.mutable.MutableInt; -import org.roaringbitmap.RoaringBitmap; +import org.apache.pulsar.common.util.collections.LongBitmap; +import org.apache.pulsar.common.util.collections.LongBitmaps; /** * Tracks the used consumer name indexes for each consumer name. @@ -39,7 +40,7 @@ * changes are minimized over time, although a better solution would be to avoid reusing the same consumer name * in the first place. * - * When a consumer is removed, the index is deallocated. RoaringBitmap is used to keep track of the used indexes. + * When a consumer is removed, the index is deallocated. LongBitmap is used to keep track of the used indexes. * The data structure to track a consumer name is removed when the reference count of the consumer name is zero. * * This class is not thread-safe and should be used in a synchronized context in the caller. @@ -56,18 +57,18 @@ record ConsumerEntry(String consumerName, int nameIndex, MutableInt refCount) { } /* - * Tracks the used indexes for a consumer name using a RoaringBitmap. + * Tracks the used indexes for a consumer name using a LongBitmap. * A specific index slot is used when the bit is set. * When all bits are cleared, the customer name can be removed from tracking. */ static class ConsumerNameIndexSlots { - private RoaringBitmap indexSlots = new RoaringBitmap(); + private LongBitmap indexSlots = LongBitmaps.create(); public int allocateIndexSlot() { // find the first index that is not set, if there is no such index, add a new one int index = (int) indexSlots.nextAbsentValue(0); if (index == -1) { - index = indexSlots.getCardinality(); + index = (int) indexSlots.cardinality(); } indexSlots.add(index); return index; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/DrainingHashesTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/DrainingHashesTracker.java index 8570e2f4d36fb..0e1d15e385e8d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/DrainingHashesTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/DrainingHashesTracker.java @@ -24,7 +24,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.PrimitiveIterator; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -33,7 +32,8 @@ import org.apache.pulsar.common.policies.data.DrainingHash; import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl; import org.apache.pulsar.common.policies.data.stats.DrainingHashImpl; -import org.roaringbitmap.RoaringBitmap; +import org.apache.pulsar.common.util.collections.LongBitmap; +import org.apache.pulsar.common.util.collections.LongBitmaps; /** * A thread-safe map to store draining hashes in the consumer. @@ -142,7 +142,7 @@ int getBlockedCount() { } private class ConsumerDrainingHashesStats { - private final RoaringBitmap drainingHashes = new RoaringBitmap(); + private final LongBitmap drainingHashes = LongBitmaps.create(); private long drainingHashesClearedTotal; private final ReentrantReadWriteLock statsLock = new ReentrantReadWriteLock(); @@ -163,11 +163,7 @@ public boolean clearHash(int hash) { boolean empty = drainingHashes.isEmpty(); if (log.isDebugEnabled()) { log.debug("[{}] Cleared hash {} in stats. empty={} totalCleared={} hashes={}", - dispatcherName, hash, empty, drainingHashesClearedTotal, drainingHashes.getCardinality()); - } - if (empty) { - // reduce memory usage by trimming the bitmap when the RoaringBitmap instance is empty - drainingHashes.trim(); + dispatcherName, hash, empty, drainingHashesClearedTotal, drainingHashes.cardinality()); } return empty; } finally { @@ -178,16 +174,15 @@ public boolean clearHash(int hash) { public void updateConsumerStats(Consumer consumer, ConsumerStatsImpl consumerStats) { statsLock.readLock().lock(); try { - int drainingHashesUnackedMessages = 0; List drainingHashesStats = new ArrayList<>(); - PrimitiveIterator.OfInt hashIterator = drainingHashes.stream().iterator(); - while (hashIterator.hasNext()) { - int hash = hashIterator.nextInt(); + int[] drainingHashesUnackedMessages = {0}; + drainingHashes.forEachLong(hashLong -> { + int hash = (int) hashLong; DrainingHashEntry entry = getEntry(hash); if (entry == null) { log.debug("[{}] Draining hash {} not found in the tracker for consumer {}", dispatcherName, hash, consumer); - continue; + return; } int unackedMessages = entry.getRefCount(); DrainingHashImpl drainingHash = new DrainingHashImpl(); @@ -195,11 +190,11 @@ public void updateConsumerStats(Consumer consumer, ConsumerStatsImpl consumerSta drainingHash.unackMsgs = unackedMessages; drainingHash.blockedAttempts = entry.getBlockedCount(); drainingHashesStats.add(drainingHash); - drainingHashesUnackedMessages += unackedMessages; - } + drainingHashesUnackedMessages[0] += unackedMessages; + }); consumerStats.drainingHashesCount = drainingHashesStats.size(); consumerStats.drainingHashesClearedTotal = drainingHashesClearedTotal; - consumerStats.drainingHashesUnackedMessages = drainingHashesUnackedMessages; + consumerStats.drainingHashesUnackedMessages = drainingHashesUnackedMessages[0]; consumerStats.drainingHashes = drainingHashesStats; } finally { statsLock.readLock().unlock(); diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index 8e1f4bdf29f77..638ed8506f3c3 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -219,6 +219,11 @@ completable-futures + + org.roaringbitmap + RoaringBitmap + + org.bouncycastle diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java new file mode 100644 index 0000000000000..7c65441004bfe --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java @@ -0,0 +1,441 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util.collections; + +import io.netty.buffer.ByteBuf; +import java.io.DataInput; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.concurrent.locks.StampedLock; +import java.util.function.LongConsumer; +import org.roaringbitmap.PeekableIntIterator; +import org.roaringbitmap.buffer.MutableRoaringBitmap; + +/** + * {@link LongBitmap} implementation backed by {@link MutableRoaringBitmap} and guarded + * by a {@link StampedLock}. + * + *

Thread-safety basis. RoaringBitmap is not thread-safe by default + * (see pulsar#25991). This + * wrapper relies on the documented contract that {@link MutableRoaringBitmap}'s read + * methods — the {@code ImmutableBitmapDataProvider} surface inherited from + * {@code ImmutableRoaringBitmap} — do not mutate internal state, while methods added by + * {@code BitmapDataProvider} and other {@code MutableRoaringBitmap} mutators + * ({@code andNot}, {@code or}, {@code checkedRemove}, {@code runOptimize}, {@code clone}, + * ...) do. Read methods run under the read lock; mutators under the write lock. + * {@code clone()} is used under the read lock in {@link #forEachLong} and {@link #serialize}; + * its source has been audited to be read-only on the live bitmap. Before upgrading + * the RoaringBitmap dependency or changing the lock split, re-audit these methods + * and run the concurrency regression tests ({@code testConcurrentForEachLongAndMutate}, + * {@code testOrDoesNotMutateInput}). + * + *

Critical sections. Single-value reads take the read lock; mutations take the + * write lock. Bulk mutations that touch two bitmaps ({@link #or}) acquire this bitmap's + * write lock and the other's read lock in {@code identityHashCode} order, so concurrent + * {@code A.or(B)} and {@code B.or(A)} cannot deadlock. Long non-mutating work + * ({@link #serialize}, {@link #forEachLong}) clones under a brief read lock and finishes + * without holding it, so optimize/iterate/runOptimize don't block writers. + * + *

Memory. {@link MutableRoaringBitmap#trim()} fires when removals since the + * last trim reach {@link #TRIM_AFTER_REMOVES}, or whenever the bitmap becomes empty. + * {@link #serialize} runs {@code runOptimize()} on the clone so persisted bytes are compact. + */ +class ConcurrentRoaringBitmap implements LongBitmap { + + private static final long TRIM_AFTER_REMOVES = 10000; + private static final long UINT32_SIZE = 1L << 32; + private static final long MAX_UINT32 = UINT32_SIZE - 1; + + private final MutableRoaringBitmap bitmap; + private final StampedLock lock; + private long removesSinceTrim; + + ConcurrentRoaringBitmap() { + this.bitmap = new MutableRoaringBitmap(); + this.lock = new StampedLock(); + } + + private ConcurrentRoaringBitmap(MutableRoaringBitmap bitmap) { + this.bitmap = bitmap; + this.lock = new StampedLock(); + } + + @Override + public void add(long value) { + validateRange(value); + long stamp = lock.writeLock(); + try { + bitmap.add((int) value); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public boolean checkedAdd(long value) { + validateRange(value); + long stamp = lock.writeLock(); + try { + return bitmap.checkedAdd((int) value); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public void add(long from, long to) { + if (to <= from) { + return; + } + validateRange(from); + validateRange(to - 1); + long stamp = lock.writeLock(); + try { + bitmap.add(from, to); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public void remove(long value) { + validateRange(value); + long stamp = lock.writeLock(); + try { + if (bitmap.checkedRemove((int) value)) { + removesSinceTrim++; + maybeTrim(); + } + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public void remove(long from, long to) { + if (to <= from) { + return; + } + validateRange(from); + validateRange(to - 1); + long stamp = lock.writeLock(); + try { + bitmap.remove(from, to); + // Range size upper-bounds removals; clamp so a huge range can't overflow the counter. + removesSinceTrim = Math.min(removesSinceTrim + (to - from), TRIM_AFTER_REMOVES); + maybeTrim(); + } finally { + lock.unlockWrite(stamp); + } + } + + @Override + public boolean contains(long value) { + if (value < 0 || value > MAX_UINT32) { + return false; + } + long stamp = lock.readLock(); + try { + return bitmap.contains((int) value); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public boolean contains(long from, long to) { + if (from < 0 || from > MAX_UINT32 || to <= from) { + return false; + } + long stamp = lock.readLock(); + try { + // Clamp: contains treats out-of-range `to` as a query past the uint32 end, but + // RoaringBitmap.contains is unreliable when `to` exceeds UINT32_SIZE. + return bitmap.contains(from, Math.min(to, UINT32_SIZE)); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public long cardinality() { + long stamp = lock.readLock(); + try { + return bitmap.getLongCardinality(); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public boolean isEmpty() { + long stamp = lock.readLock(); + try { + return bitmap.isEmpty(); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public long nextAbsentValue(long from) { + if (from < 0 || from > MAX_UINT32) { + return -1; + } + long stamp = lock.readLock(); + try { + return bitmap.nextAbsentValue((int) from); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public void or(LongBitmap other) { + if (other == this) { + return; + } + if (!(other instanceof ConcurrentRoaringBitmap)) { + throw new IllegalArgumentException("Unsupported LongBitmap type: " + other.getClass()); + } + ConcurrentRoaringBitmap that = (ConcurrentRoaringBitmap) other; + + // Acquire this.writeLock + that.readLock in identityHashCode order so concurrent + // A.or(B) and B.or(A) don't deadlock. Fall back to inner bitmap identity on collision. + boolean thisFirst; + int outerCmp = Integer.compare( + System.identityHashCode(this), System.identityHashCode(that)); + if (outerCmp != 0) { + thisFirst = outerCmp < 0; + } else { + thisFirst = System.identityHashCode(this.bitmap) < System.identityHashCode(that.bitmap); + } + + if (thisFirst) { + long thisStamp = this.lock.writeLock(); + try { + long thatStamp = that.lock.readLock(); + try { + this.bitmap.or(that.bitmap); + } finally { + that.lock.unlockRead(thatStamp); + } + } finally { + this.lock.unlockWrite(thisStamp); + } + } else { + long thatStamp = that.lock.readLock(); + try { + long thisStamp = this.lock.writeLock(); + try { + this.bitmap.or(that.bitmap); + } finally { + this.lock.unlockWrite(thisStamp); + } + } finally { + that.lock.unlockRead(thatStamp); + } + } + } + + @Override + public void forEachLong(LongConsumer action) { + MutableRoaringBitmap snapshot; + long stamp = lock.readLock(); + try { + snapshot = bitmap.clone(); + } finally { + lock.unlockRead(stamp); + } + snapshot.forEach((org.roaringbitmap.IntConsumer) v -> + action.accept(Integer.toUnsignedLong(v))); + } + + @Override + public long drainTo(long limit, LongConsumer action) { + if (limit <= 0) { + return 0; + } + MutableRoaringBitmap toRemove = new MutableRoaringBitmap(); + long collected; + long writeStamp = lock.writeLock(); + try { + PeekableIntIterator it = bitmap.getIntIterator(); + collected = 0; + while (collected < limit && it.hasNext()) { + toRemove.add(it.next()); + collected++; + } + if (collected == 0) { + return 0; + } + bitmap.andNot(toRemove); + removesSinceTrim = Math.min(removesSinceTrim + collected, TRIM_AFTER_REMOVES); + maybeTrim(); + } finally { + lock.unlockWrite(writeStamp); + } + + toRemove.forEach((org.roaringbitmap.IntConsumer) v -> + action.accept(Integer.toUnsignedLong(v))); + return collected; + } + + @Override + public long serializedSize() { + long stamp = lock.readLock(); + try { + return bitmap.serializedSizeInBytes(); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public byte[] serialize() { + MutableRoaringBitmap copy; + long stamp = lock.readLock(); + try { + copy = bitmap.clone(); + } finally { + lock.unlockRead(stamp); + } + copy.runOptimize(); + byte[] bytes = new byte[copy.serializedSizeInBytes()]; + copy.serialize(ByteBuffer.wrap(bytes)); + return bytes; + } + + static ConcurrentRoaringBitmap deserialize(ByteBuf buf) { + try { + ByteBuffer nioBuffer = buf.nioBuffer(buf.readerIndex(), buf.readableBytes()); + int startPosition = nioBuffer.position(); + MutableRoaringBitmap bitmap = new MutableRoaringBitmap(); + bitmap.deserialize(new ByteBufferDataInput(nioBuffer)); + buf.skipBytes(nioBuffer.position() - startPosition); + return new ConcurrentRoaringBitmap(bitmap); + } catch (IOException e) { + throw new RuntimeException("Failed to deserialize LongBitmap", e); + } + } + + /** + * Trims the underlying bitmap if enough removals have accumulated or it's empty. + * Caller must hold the write lock and have already updated {@link #removesSinceTrim}. + */ + private void maybeTrim() { + if (removesSinceTrim >= TRIM_AFTER_REMOVES || bitmap.isEmpty()) { + bitmap.trim(); + removesSinceTrim = 0; + } + } + + private static void validateRange(long value) { + if (value < 0 || value > MAX_UINT32) { + throw new IllegalArgumentException( + "Value out of range [0, " + MAX_UINT32 + "]: " + value); + } + } + + /** Minimal {@link DataInput} over a {@link ByteBuffer} for RoaringBitmap deserialization. */ + private static final class ByteBufferDataInput implements DataInput { + private final ByteBuffer buffer; + + ByteBufferDataInput(ByteBuffer buffer) { + this.buffer = buffer; + } + + @Override + public void readFully(byte[] b) { + buffer.get(b); + } + + @Override + public void readFully(byte[] b, int off, int len) { + buffer.get(b, off, len); + } + + @Override + public int skipBytes(int n) { + int skip = Math.min(n, buffer.remaining()); + buffer.position(buffer.position() + skip); + return skip; + } + + @Override + public boolean readBoolean() { + return buffer.get() != 0; + } + + @Override + public byte readByte() { + return buffer.get(); + } + + @Override + public int readUnsignedByte() { + return Byte.toUnsignedInt(buffer.get()); + } + + @Override + public short readShort() { + return buffer.getShort(); + } + + @Override + public int readUnsignedShort() { + return Short.toUnsignedInt(buffer.getShort()); + } + + @Override + public char readChar() { + return buffer.getChar(); + } + + @Override + public int readInt() { + return buffer.getInt(); + } + + @Override + public long readLong() { + return buffer.getLong(); + } + + @Override + public float readFloat() { + return buffer.getFloat(); + } + + @Override + public double readDouble() { + return buffer.getDouble(); + } + + @Override + public String readLine() { + throw new UnsupportedOperationException(); + } + + @Override + public String readUTF() { + throw new UnsupportedOperationException(); + } + } +} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongBitmap.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongBitmap.java new file mode 100644 index 0000000000000..5176da8c12598 --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongBitmap.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util.collections; + +import java.util.function.LongConsumer; + +/** + * Thread-safe bitmap abstraction for tracking long values. + * + *

The current implementation supports values in the unsigned 32-bit range + * {@code [0, 2^32 - 1]}. Methods that modify the bitmap reject values outside this + * range with {@link IllegalArgumentException}. Query methods return {@code false} + * or {@code -1} for out-of-range values where applicable. + * + *

Supports point and range operations, bulk union, atomic draining, iteration, + * and serialization. All operations are thread-safe. + * + *

This abstraction is used for high-throughput broker metadata tracking, + * including delayed-delivery tracking, consumer-name allocation, and + * draining-hash tracking. + * + *

Example: + * 

{@code
+ * LongBitmap bitmap = LongBitmaps.create();
+ * bitmap.add(12345L);
+ * if (bitmap.contains(12345L)) { ... }
+ *
+ * byte[] bytes = bitmap.serialize();
+ * LongBitmap restored = LongBitmaps.deserialize(Unpooled.wrappedBuffer(bytes));
+ * }
+ */ +public interface LongBitmap { + + /** + * Adds a value. + * + * @param value value to add, must be in {@code [0, 2^32 - 1]} + * @throws IllegalArgumentException if value is outside the supported range + */ + void add(long value); + + /** + * Adds a value if it is not already present. + * + *

This operation is atomic. Unlike {@code if (!contains(value)) add(value)}, + * the check and add are performed as a single operation. + * + * @param value value to add, must be in {@code [0, 2^32 - 1]} + * @return {@code true} if the value was added, {@code false} if it already existed + * @throws IllegalArgumentException if value is outside the supported range + */ + boolean checkedAdd(long value); + + /** + * Adds all values in the half-open range {@code [from, to)}. + * + *

No-op if {@code to <= from}. + * + * @param from inclusive lower bound + * @param to exclusive upper bound + * @throws IllegalArgumentException if the range exceeds the supported value range + */ + void add(long from, long to); + + /** + * Removes a value. No-op if absent. + * + * @param value value to remove + * @throws IllegalArgumentException if value is outside the supported range + */ + void remove(long value); + + /** + * Removes all values in the half-open range {@code [from, to)}. + * + *

No-op if {@code to <= from}. + * + * @param from inclusive lower bound + * @param to exclusive upper bound + * @throws IllegalArgumentException if the range exceeds the supported value range + */ + void remove(long from, long to); + + /** + * Returns whether the bitmap contains the given value. + * + * @param value value to check + * @return {@code true} if present, otherwise {@code false} + */ + boolean contains(long value); + + /** + * Returns whether all values in {@code [from, to)} are present. + * + * @param from inclusive lower bound + * @param to exclusive upper bound + * @return {@code true} if all values in the range are present + */ + boolean contains(long from, long to); + + /** Returns the number of values currently stored. */ + long cardinality(); + + /** Returns {@code true} if no values are stored. */ + boolean isEmpty(); + + /** + * Returns the smallest absent value greater than or equal to {@code from}. + * + * @param from inclusive lower bound + * @return next absent value, or {@code -1} if none exists + */ + long nextAbsentValue(long from); + + /** + * Adds all values from {@code other} into this bitmap. + * + * @param other bitmap to merge + */ + void or(LongBitmap other); + + /** + * Iterates values in ascending order. + * + *

The iteration observes a stable view of the bitmap. Implementations may + * choose the mechanism used to provide this guarantee. + * + * @param action callback invoked for each value + */ + void forEachLong(LongConsumer action); + + /** + * Atomically removes up to {@code limit} values and invokes {@code action} + * for each removed value. + * + *

Selection and removal are performed atomically. The callback is invoked + * after removal has completed. + * + * @param limit maximum number of values to drain + * @param action callback invoked for each removed value + * @return number of values drained + */ + long drainTo(long limit, LongConsumer action); + + /** + * Returns an upper bound of the serialized size. + */ + long serializedSize(); + + /** + * Serializes the bitmap into a newly allocated byte array. + */ + byte[] serialize(); +} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongBitmaps.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongBitmaps.java new file mode 100644 index 0000000000000..489502a439d4a --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongBitmaps.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util.collections; + +import io.netty.buffer.ByteBuf; + +/** + * Factory for creating {@link LongBitmap} instances. + */ +public final class LongBitmaps { + + private LongBitmaps() { + // utility class + } + + /** + * Creates a new empty thread-safe LongBitmap. + * + * @return a new LongBitmap instance + */ + public static LongBitmap create() { + return new ConcurrentRoaringBitmap(); + } + + /** + * Deserializes a LongBitmap from a ByteBuf. + * + *

Advances the buffer's {@code readerIndex} by the number of bytes consumed. The + * buffer may be heap-backed, direct, or a {@link io.netty.buffer.CompositeByteBuf} — + * the implementation reads via {@link ByteBuf#nioBuffer(int, int)} without copying + * when possible. + * + *

The serialized format is the standard 32-bit RoaringBitmap portable format, so + * buffers produced by {@link LongBitmap#serialize()} round-trip exactly. Buffers in + * other formats (e.g. {@code Roaring64Bitmap}) are rejected. + * + * @param buf the input buffer positioned at the start of the serialized bitmap + * @return the deserialized LongBitmap + * @throws RuntimeException if the buffer is malformed, truncated, or in an + * unrecognized format (wraps the underlying {@link java.io.IOException}) + */ + public static LongBitmap deserialize(ByteBuf buf) { + return ConcurrentRoaringBitmap.deserialize(buf); + } +} diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/LongBitmapCompatibilityTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/LongBitmapCompatibilityTest.java new file mode 100644 index 0000000000000..6c9081db44f52 --- /dev/null +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/LongBitmapCompatibilityTest.java @@ -0,0 +1,235 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util.collections; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.nio.ByteBuffer; +import org.roaringbitmap.RoaringBitmap; +import org.roaringbitmap.longlong.Roaring64Bitmap; +import org.testng.annotations.Test; + +/** + * Verifies the serialization compatibility characteristics of {@link LongBitmap}. + * + *

Context: the previous implementation used two different RoaringBitmap variants: + *

    + *
  • {@code InMemoryDelayedDeliveryTracker} used {@link Roaring64Bitmap} (in-memory only) + *
  • {@code BucketDelayedDeliveryTracker} used {@link RoaringBitmap} (32-bit, persisted) + *
+ * + *

The new {@link LongBitmap} abstraction always uses the 32-bit {@link RoaringBitmap} + * internally. These tests document and verify: + *

    + *
  1. 32-bit buffer compatibility: {@link LongBitmap} buffers are byte-identical to standard + * {@link RoaringBitmap} buffers (required for {@code BucketDelayedDeliveryTracker} + * backward compatibility — old persisted snapshots must round-trip). + *
  2. 64-bit buffer incompatibility: {@link Roaring64Bitmap} buffers cannot be deserialized + * by {@link LongBitmap} (format includes high-32-bit bucket prefixes). Migration from + * {@code Roaring64Bitmap} is safe only for in-memory state; persisted state would require + * a one-time entry-by-entry rebuild. + *
  3. Behavioral equivalence within {@code uint32} range: {@link LongBitmap} and + * {@link Roaring64Bitmap} produce identical add/remove/contains results for all + * values in {@code [0, 2^32)}. + *
+ */ +public class LongBitmapCompatibilityTest { + + /** + * LongBitmap's serialized bytes must be identical to a 32-bit RoaringBitmap's + * for the same uint32 values. This guarantees persisted snapshots written by + * the old {@code BucketDelayedDeliveryTracker} (using RoaringBitmap) remain + * readable, and vice versa. + */ + @Test + public void testLongBitmapBufferEqualsStandardRoaringBitmap() throws Exception { + LongBitmap longBitmap = LongBitmaps.create(); + longBitmap.add(0); + longBitmap.add(100); + longBitmap.add(1L << 20); + longBitmap.add(0xFFFFFFFFL); // uint32 max + + RoaringBitmap roaring = new RoaringBitmap(); + roaring.add(0); + roaring.add(100); + roaring.add(1 << 20); + roaring.add(0xFFFFFFFF); // -1 as signed int + + byte[] longBitmapBytes = serializeLongBitmap(longBitmap); + byte[] roaringBytes = serializeRoaring32(roaring); + + assertEquals(longBitmapBytes, roaringBytes, + "LongBitmap buffer must be byte-identical to standard 32-bit RoaringBitmap buffer"); + } + + /** + * Round-trip: 32-bit RoaringBitmap buffer -> LongBitmap. Confirms that + * persisted data written by the old code path can be read by LongBitmap. + */ + @Test + public void testDeserializeFromRoaring32Buffer() throws Exception { + RoaringBitmap roaring = new RoaringBitmap(); + for (int i = 0; i < 1000; i += 7) { + roaring.add(i); + } + roaring.add(0xFFFFFFFF); + + byte[] roaringBytes = serializeRoaring32(roaring); + ByteBuf buf = Unpooled.wrappedBuffer(roaringBytes); + try { + LongBitmap longBitmap = LongBitmaps.deserialize(buf); + assertEquals(longBitmap.cardinality(), roaring.getLongCardinality()); + assertTrue(longBitmap.contains(0xFFFFFFFFL)); + for (int i = 0; i < 1000; i += 7) { + assertTrue(longBitmap.contains(i), "missing " + i); + } + for (int i = 1; i < 1000; i += 7) { + assertFalse(longBitmap.contains(i)); + } + } finally { + buf.release(); + } + } + + /** + * LongBitmap buffer cannot be deserialized as a Roaring64Bitmap. + * The 32-bit RoaringBitmap portable format (cookie 12346, ~22 bytes for small sets) + * is shorter than Roaring64Bitmap's bucketed header, so {@code Roaring64Bitmap.deserialize} + * throws {@link java.nio.BufferUnderflowException}. + * + *

Migration implication: persisted 32-bit data cannot be read by old code paths + * that still expect Roaring64Bitmap, and vice versa. + */ + @Test + public void testLongBitmapBufferNotReadableAsRoaring64() throws Exception { + LongBitmap longBitmap = LongBitmaps.create(); + longBitmap.add(1); + longBitmap.add(100); + longBitmap.add(1000); + + byte[] longBitmapBytes = serializeLongBitmap(longBitmap); + + Roaring64Bitmap roaring64 = new Roaring64Bitmap(); + // 32-bit format is shorter than 64-bit header expects — buffer underflow or related I/O error. + assertThrows(Exception.class, + () -> roaring64.deserialize(ByteBuffer.wrap(longBitmapBytes))); + } + + /** + * Roaring64Bitmap buffer cannot be deserialized as a LongBitmap. + * Roaring64Bitmap's serialized format starts with a bucket count, not the + * 32-bit cookie (12346), so {@code MutableRoaringBitmap.deserialize} throws. + */ + @Test + public void testRoaring64BufferNotReadableAsLongBitmap() throws Exception { + Roaring64Bitmap roaring64 = new Roaring64Bitmap(); + roaring64.addLong(1); + roaring64.addLong(100); + roaring64.addLong(1000); + + byte[] roaring64Bytes = serializeRoaring64(roaring64); + + ByteBuf buf = Unpooled.wrappedBuffer(roaring64Bytes); + try { + // MutableRoaringBitmap wraps IOException as RuntimeException. + assertThrows(Exception.class, () -> LongBitmaps.deserialize(buf)); + } finally { + buf.release(); + } + } + + /** + * Behavioral equivalence within uint32 range: LongBitmap and Roaring64Bitmap + * produce identical results for all operations on values in [0, 2^32). + * This is what makes the InMemoryDelayedDeliveryTracker migration safe — + * BookKeeper entry IDs are currently always < 2^32. + */ + @Test + public void testBehavioralEquivalenceWithinUint32() { + LongBitmap longBitmap = LongBitmaps.create(); + Roaring64Bitmap roaring64 = new Roaring64Bitmap(); + + long[] values = {0, 1, 100, 65535, 65536, 1L << 20, 1L << 30, 0xFFFFFFFFL}; + for (long v : values) { + longBitmap.add(v); + roaring64.addLong(v); + } + + assertEquals(longBitmap.cardinality(), roaring64.getLongCardinality()); + for (long v : values) { + assertTrue(longBitmap.contains(v)); + assertTrue(roaring64.contains(v)); + } + + long[] toRemove = {0, 100, 65536, 1L << 30}; + for (long v : toRemove) { + longBitmap.remove(v); + roaring64.removeLong(v); + } + + assertEquals(longBitmap.cardinality(), roaring64.getLongCardinality()); + for (long v : toRemove) { + assertFalse(longBitmap.contains(v)); + assertFalse(roaring64.contains(v)); + } + } + + /** + * LongBitmap accepts the uint32 boundary (2^32 - 1) but rejects 2^32 and above. + * Migration from Roaring64Bitmap must ensure no values >= 2^32 are present; + * otherwise the migration would silently drop or reject those entries. + */ + @Test + public void testUint32Boundary() { + LongBitmap longBitmap = LongBitmaps.create(); + longBitmap.add(0xFFFFFFFFL); + assertTrue(longBitmap.contains(0xFFFFFFFFL)); + assertEquals(longBitmap.cardinality(), 1); + + assertThrows(IllegalArgumentException.class, () -> longBitmap.add(0x100000000L)); + assertThrows(IllegalArgumentException.class, () -> longBitmap.add(-1)); + } + + private static byte[] serializeLongBitmap(LongBitmap bitmap) { + return bitmap.serialize(); + } + + private static byte[] serializeRoaring32(RoaringBitmap bitmap) throws Exception { + bitmap.runOptimize(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(baos); + bitmap.serialize(dos); + dos.close(); + return baos.toByteArray(); + } + + private static byte[] serializeRoaring64(Roaring64Bitmap bitmap) throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(baos); + bitmap.serialize(dos); + dos.close(); + return baos.toByteArray(); + } +} diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/LongBitmapTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/LongBitmapTest.java new file mode 100644 index 0000000000000..35bbf9b5f57b0 --- /dev/null +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/LongBitmapTest.java @@ -0,0 +1,824 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util.collections; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.testng.annotations.Test; + +/** + * Unit tests for {@link LongBitmap}. Covers point/range operations, uint32 boundary + * behavior, serialization round-trip, drain-to atomicity, and concurrency contracts + * (or lock ordering, snapshot safety under mutation, or-input immutability). + */ +public class LongBitmapTest { + + @Test + public void testBasicOperations() { + LongBitmap bitmap = LongBitmaps.create(); + + assertEquals(bitmap.cardinality(), 0); + assertFalse(bitmap.contains(1)); + + bitmap.add(1); + bitmap.add(100); + bitmap.add(1000); + + assertEquals(bitmap.cardinality(), 3); + assertTrue(bitmap.contains(1)); + assertTrue(bitmap.contains(100)); + assertTrue(bitmap.contains(1000)); + assertFalse(bitmap.contains(2)); + + bitmap.remove(100); + assertEquals(bitmap.cardinality(), 2); + assertFalse(bitmap.contains(100)); + } + + @Test + public void testRangeValidation() { + LongBitmap bitmap = LongBitmaps.create(); + bitmap.add(0); + bitmap.add(0xFFFFFFFFL); + assertTrue(bitmap.contains(0)); + assertTrue(bitmap.contains(0xFFFFFFFFL)); + + assertThrows(IllegalArgumentException.class, () -> bitmap.add(-1)); + assertThrows(IllegalArgumentException.class, () -> bitmap.add(0x100000000L)); + assertFalse(bitmap.contains(-1)); + assertFalse(bitmap.contains(0x100000000L)); + } + + @Test + public void testSerialization() { + LongBitmap bitmap = LongBitmaps.create(); + for (int i = 0; i < 1000; i += 10) { + bitmap.add(i); + } + + byte[] bytes = bitmap.serialize(); + assertTrue(bytes.length > 0); + + LongBitmap deserialized = LongBitmaps.deserialize(Unpooled.wrappedBuffer(bytes)); + assertEquals(deserialized.cardinality(), 100); + for (int i = 0; i < 1000; i += 10) { + assertTrue(deserialized.contains(i)); + } + assertFalse(deserialized.contains(5)); + } + + @Test + public void testDeserializeFromVariousByteBufTypes() { + LongBitmap bitmap = LongBitmaps.create(); + for (int i = 0; i < 2000; i += 7) { + bitmap.add(i); + } + byte[] bytes = bitmap.serialize(); + + ByteBuf heap = Unpooled.wrappedBuffer(bytes); + try { + LongBitmap r = LongBitmaps.deserialize(heap); + assertEquals(r.cardinality(), bitmap.cardinality()); + assertEquals(heap.readerIndex(), bytes.length); + } finally { + heap.release(); + } + + ByteBuf direct = Unpooled.directBuffer(bytes.length); + try { + direct.writeBytes(bytes); + LongBitmap r = LongBitmaps.deserialize(direct); + assertEquals(r.cardinality(), bitmap.cardinality()); + assertEquals(direct.readerIndex(), bytes.length); + } finally { + direct.release(); + } + + ByteBuf singleComp = Unpooled.wrappedBuffer(new ByteBuf[]{Unpooled.wrappedBuffer(bytes)}); + try { + LongBitmap r = LongBitmaps.deserialize(singleComp); + assertEquals(r.cardinality(), bitmap.cardinality()); + assertEquals(singleComp.readerIndex(), bytes.length); + } finally { + singleComp.release(); + } + + int split = bytes.length / 2; + ByteBuf part1 = Unpooled.wrappedBuffer(bytes, 0, split); + ByteBuf part2 = Unpooled.wrappedBuffer(bytes, split, bytes.length - split); + ByteBuf composite = Unpooled.wrappedBuffer(part1, part2); + try { + LongBitmap r = LongBitmaps.deserialize(composite); + assertEquals(r.cardinality(), bitmap.cardinality()); + assertEquals(composite.readerIndex(), bytes.length); + } finally { + composite.release(); + } + } + + @Test + public void testAddIsIdempotent() { + LongBitmap bitmap = LongBitmaps.create(); + bitmap.add(42); + assertEquals(bitmap.cardinality(), 1); + assertTrue(bitmap.contains(42)); + + bitmap.add(42); + bitmap.add(42); + assertEquals(bitmap.cardinality(), 1); + assertTrue(bitmap.contains(42)); + } + + @Test + public void testCheckedAdd() { + LongBitmap bitmap = LongBitmaps.create(); + + assertTrue(bitmap.checkedAdd(42)); + assertTrue(bitmap.contains(42)); + assertEquals(bitmap.cardinality(), 1); + + assertFalse(bitmap.checkedAdd(42)); + assertEquals(bitmap.cardinality(), 1); + + assertTrue(bitmap.checkedAdd(100)); + assertEquals(bitmap.cardinality(), 2); + + assertThrows(IllegalArgumentException.class, () -> bitmap.checkedAdd(-1)); + assertThrows(IllegalArgumentException.class, () -> bitmap.checkedAdd(0x100000000L)); + assertEquals(bitmap.cardinality(), 2); + } + + @Test + public void testForEach() { + LongBitmap bitmap = LongBitmaps.create(); + bitmap.add(1); + bitmap.add(5); + bitmap.add(10); + + List values = new ArrayList<>(); + bitmap.forEachLong(values::add); + + assertEquals(values.size(), 3); + Collections.sort(values); + assertEquals(values.get(0).longValue(), 1); + assertEquals(values.get(1).longValue(), 5); + assertEquals(values.get(2).longValue(), 10); + } + + @Test + public void testConcurrentReads() throws Exception { + LongBitmap bitmap = LongBitmaps.create(); + for (int i = 0; i < 10000; i++) { + bitmap.add(i); + } + + int numThreads = 10; + ExecutorService executor = Executors.newFixedThreadPool(numThreads); + CountDownLatch latch = new CountDownLatch(numThreads); + AtomicInteger errors = new AtomicInteger(0); + + for (int i = 0; i < numThreads; i++) { + executor.submit(() -> { + try { + for (int j = 0; j < 1000; j++) { + if (!bitmap.contains(j)) { + errors.incrementAndGet(); + } + if (bitmap.cardinality() != 10000) { + errors.incrementAndGet(); + } + } + } finally { + latch.countDown(); + } + }); + } + + assertTrue(latch.await(30, TimeUnit.SECONDS)); + executor.shutdown(); + assertEquals(errors.get(), 0); + } + + @Test + public void testConcurrentWrites() throws Exception { + LongBitmap bitmap = LongBitmaps.create(); + + int numThreads = 10; + int valuesPerThread = 1000; + ExecutorService executor = Executors.newFixedThreadPool(numThreads); + CountDownLatch latch = new CountDownLatch(numThreads); + + for (int t = 0; t < numThreads; t++) { + int threadId = t; + executor.submit(() -> { + try { + for (int i = 0; i < valuesPerThread; i++) { + bitmap.add(threadId * valuesPerThread + i); + } + } finally { + latch.countDown(); + } + }); + } + + assertTrue(latch.await(30, TimeUnit.SECONDS)); + executor.shutdown(); + + assertEquals(bitmap.cardinality(), numThreads * valuesPerThread); + for (int t = 0; t < numThreads; t++) { + for (int i = 0; i < valuesPerThread; i++) { + assertTrue(bitmap.contains(t * valuesPerThread + i)); + } + } + } + + @Test + public void testConcurrentReadWrite() throws Exception { + LongBitmap bitmap = LongBitmaps.create(); + for (int i = 0; i < 5000; i++) { + bitmap.add(i); + } + + int numReaders = 5; + int numWriters = 5; + ExecutorService executor = Executors.newFixedThreadPool(numReaders + numWriters); + CountDownLatch latch = new CountDownLatch(numReaders + numWriters); + AtomicInteger errors = new AtomicInteger(0); + + for (int i = 0; i < numReaders; i++) { + executor.submit(() -> { + try { + for (int j = 0; j < 1000; j++) { + bitmap.contains(j % 5000); + bitmap.cardinality(); + } + } catch (Exception e) { + errors.incrementAndGet(); + } finally { + latch.countDown(); + } + }); + } + for (int i = 0; i < numWriters; i++) { + int writerId = i; + executor.submit(() -> { + try { + for (int j = 0; j < 1000; j++) { + bitmap.add(5000 + writerId * 1000 + j); + } + } catch (Exception e) { + errors.incrementAndGet(); + } finally { + latch.countDown(); + } + }); + } + + assertTrue(latch.await(30, TimeUnit.SECONDS)); + executor.shutdown(); + assertEquals(errors.get(), 0); + } + + @Test + public void testConcurrentForEachLongAndMutate() throws Exception { + // forEachLong takes a clone() snapshot under read lock; concurrent mutations on the + // live bitmap must never corrupt the snapshot. Regression guard for pulsar#25991. + LongBitmap bitmap = LongBitmaps.create(); + for (int i = 0; i < 10000; i++) { + bitmap.add(i); + } + + int numReaders = 5; + int numWriters = 5; + ExecutorService executor = Executors.newFixedThreadPool(numReaders + numWriters); + CountDownLatch latch = new CountDownLatch(numReaders + numWriters); + AtomicInteger errors = new AtomicInteger(0); + + for (int i = 0; i < numReaders; i++) { + executor.submit(() -> { + try { + for (int j = 0; j < 100; j++) { + long[] last = {-1}; + bitmap.forEachLong(v -> { + // Snapshot values must arrive in ascending order. + if (v <= last[0]) { + errors.incrementAndGet(); + } + last[0] = v; + }); + } + } catch (Exception e) { + errors.incrementAndGet(); + } finally { + latch.countDown(); + } + }); + } + + for (int i = 0; i < numWriters; i++) { + int id = i; + executor.submit(() -> { + try { + for (int j = 0; j < 1000; j++) { + long v = 10000 + id * 1000 + (j % 1000); + bitmap.add(v); + bitmap.remove(v); + } + } catch (Exception e) { + errors.incrementAndGet(); + } finally { + latch.countDown(); + } + }); + } + + assertTrue(latch.await(30, TimeUnit.SECONDS)); + executor.shutdown(); + assertEquals(errors.get(), 0); + assertEquals(bitmap.cardinality(), 10000); + } + + @Test + public void testMemoryTrim() { + LongBitmap bitmap = LongBitmaps.create(); + for (int i = 0; i < 100000; i++) { + bitmap.add(i); + } + for (int i = 0; i < 50000; i++) { + bitmap.remove(i); + } + + assertEquals(bitmap.cardinality(), 50000); + for (int i = 0; i < 50000; i++) { + assertFalse(bitmap.contains(i)); + } + for (int i = 50000; i < 100000; i++) { + assertTrue(bitmap.contains(i)); + } + } + + @Test + public void testIsEmpty() { + LongBitmap bitmap = LongBitmaps.create(); + assertTrue(bitmap.isEmpty()); + + bitmap.add(1); + assertFalse(bitmap.isEmpty()); + + bitmap.remove(1); + assertTrue(bitmap.isEmpty()); + + bitmap.remove(999); // never added + assertTrue(bitmap.isEmpty()); + } + + @Test + public void testOr() { + LongBitmap a = LongBitmaps.create(); + a.add(1); + a.add(100); + a.add(1000); + + LongBitmap b = LongBitmaps.create(); + b.add(100); // overlap + b.add(2000); // unique to b + + a.or(b); + + assertEquals(a.cardinality(), 4); + assertTrue(a.contains(1)); + assertTrue(a.contains(100)); + assertTrue(a.contains(1000)); + assertTrue(a.contains(2000)); + + // b is not modified + assertEquals(b.cardinality(), 2); + } + + @Test + public void testOrEmpty() { + LongBitmap a = LongBitmaps.create(); + a.add(1); + a.add(2); + + LongBitmap empty = LongBitmaps.create(); + a.or(empty); + assertEquals(a.cardinality(), 2); + + LongBitmap target = LongBitmaps.create(); + target.or(a); + assertEquals(target.cardinality(), 2); + } + + @Test + public void testOrSelfIsNoOp() { + LongBitmap a = LongBitmaps.create(); + a.add(1); + a.add(100); + + a.or(a); // should not deadlock + + assertEquals(a.cardinality(), 2); + assertTrue(a.contains(1)); + assertTrue(a.contains(100)); + } + + @Test + public void testRangeAddRemoveContains() { + LongBitmap bitmap = LongBitmaps.create(); + bitmap.add(100, 200); + assertEquals(bitmap.cardinality(), 100); + for (long v = 100; v < 200; v++) { + assertTrue(bitmap.contains(v)); + } + assertFalse(bitmap.contains(99)); + assertFalse(bitmap.contains(200)); + + // Single-value range [x, x+1) is equivalent to add(x). + LongBitmap single = LongBitmaps.create(); + single.add(42, 43); + assertTrue(single.contains(42, 43)); + assertTrue(single.contains(42)); + assertEquals(single.cardinality(), 1); + + // Range contains: true iff EVERY value in [from, to) is set. + assertTrue(bitmap.contains(100, 200)); + assertTrue(bitmap.contains(150, 160)); + assertFalse(bitmap.contains(99, 101)); + assertFalse(bitmap.contains(199, 201)); + assertFalse(bitmap.contains(200, 300)); + + bitmap.remove(100, 150); + assertEquals(bitmap.cardinality(), 50); + for (long v = 100; v < 150; v++) { + assertFalse(bitmap.contains(v)); + } + for (long v = 150; v < 200; v++) { + assertTrue(bitmap.contains(v)); + } + } + + @Test + public void testRangeVsSingleValueEquivalence() { + // For any v, add(v, v+1) / contains(v, v+1) / remove(v, v+1) ≡ add(v) / contains(v) / remove(v). + long[] values = {0, 1, 100, 65535, 65536, 1L << 30, 0xFFFFFFFFL}; + for (long v : values) { + LongBitmap bitmap = LongBitmaps.create(); + bitmap.add(v); + assertTrue(bitmap.contains(v)); + assertEquals(bitmap.cardinality(), 1); + + bitmap.add(v); // idempotent + assertEquals(bitmap.cardinality(), 1); + + bitmap.remove(v); + assertFalse(bitmap.contains(v)); + assertEquals(bitmap.cardinality(), 0); + } + } + + @Test + public void testNextAbsentValue() { + LongBitmap bitmap = LongBitmaps.create(); + // Empty bitmap: first absent is 0 + assertEquals(bitmap.nextAbsentValue(0), 0); + + bitmap.add(0); + bitmap.add(1); + bitmap.add(2); + // [0,1,2] present, next absent from 0 is 3 + assertEquals(bitmap.nextAbsentValue(0), 3); + + bitmap.add(5); + // Gap at 3,4 + assertEquals(bitmap.nextAbsentValue(0), 3); + assertEquals(bitmap.nextAbsentValue(3), 3); + assertEquals(bitmap.nextAbsentValue(5), 6); + + // Out of range returns -1 + assertEquals(bitmap.nextAbsentValue(-1), -1); + assertEquals(bitmap.nextAbsentValue(0x100000000L), -1); + } + + @Test + public void testDrainTo() { + LongBitmap bitmap = LongBitmaps.create(); + for (int i = 0; i < 100; i++) { + bitmap.add(i * 10); // 0, 10, 20, ..., 990 + } + + List drained = new ArrayList<>(); + long count = bitmap.drainTo(5, drained::add); + + assertEquals(count, 5); + assertEquals(drained.size(), 5); + assertEquals(drained.get(0).longValue(), 0); + assertEquals(drained.get(4).longValue(), 40); + assertEquals(bitmap.cardinality(), 95); + assertFalse(bitmap.contains(0)); + assertFalse(bitmap.contains(40)); + assertTrue(bitmap.contains(50)); + + drained.clear(); + count = bitmap.drainTo(1000, drained::add); // drain all remaining + assertEquals(count, 95); + assertTrue(bitmap.isEmpty()); + } + + @Test + public void testDrainToZeroOrNegative() { + LongBitmap bitmap = LongBitmaps.create(); + bitmap.add(1); + bitmap.add(2); + + assertEquals(bitmap.drainTo(0, v -> { + }), 0); + assertEquals(bitmap.cardinality(), 2); + + assertEquals(bitmap.drainTo(-1, v -> { + }), 0); + assertEquals(bitmap.cardinality(), 2); + } + + @Test + public void testEmptyRangeIsNoOp() { + // add(x, x) and remove(x, x) must be no-ops, not throw. + LongBitmap bitmap = LongBitmaps.create(); + bitmap.add(1); + bitmap.add(2); + + bitmap.add(5, 5); + bitmap.add(10, 1); // reversed range + assertEquals(bitmap.cardinality(), 2); + + bitmap.remove(5, 5); + bitmap.remove(10, 1); + assertEquals(bitmap.cardinality(), 2); + } + + @Test + public void testDrainToFollowedByDrainToDoesNotLeak() { + // drainTo must respect the same trim threshold as remove(value). + // Add many values, drain in small batches; if trim never fires, the bitmap + // accumulates unused container capacity across calls. + LongBitmap bitmap = LongBitmaps.create(); + for (int i = 0; i < 50000; i++) { + bitmap.add(i); + } + + // Drain 100 at a time — well under TRIM_AFTER_REMOVES (10000) per call, + // but cumulative trim counter should cross the threshold after several batches. + long totalDrained = 0; + while (!bitmap.isEmpty()) { + totalDrained += bitmap.drainTo(100, v -> { + }); + } + assertEquals(totalDrained, 50000); + assertTrue(bitmap.isEmpty()); + } + + @Test + public void testOrDoesNotMutateInput() { + // A.or(B) must treat B as read-only — required for our lock split + // (this=writeLock, other=readLock). + LongBitmap a = LongBitmaps.create(); + LongBitmap b = LongBitmaps.create(); + for (int i = 0; i < 1000; i++) { + a.add(i * 2); + b.add(i * 2 + 1); + } + long bCardinalityBefore = b.cardinality(); + + a.or(b); + + assertEquals(b.cardinality(), bCardinalityBefore); + for (int i = 0; i < 1000; i++) { + assertTrue(b.contains(i * 2 + 1), "b lost value after a.or(b)"); + assertFalse(b.contains(i * 2), "b gained value after a.or(b)"); + } + assertEquals(a.cardinality(), 2000); + } + + @Test + public void testOrCrossDirectionNoDeadlock() throws Exception { + // Concurrent A.or(B) and B.or(A) must not deadlock. + // Lock ordering by identityHashCode prevents the classic AB-BA deadlock. + int pairs = 10; + ExecutorService executor = Executors.newFixedThreadPool(4); + CountDownLatch latch = new CountDownLatch(pairs * 2); + AtomicInteger errors = new AtomicInteger(0); + + for (int i = 0; i < pairs; i++) { + final LongBitmap a = LongBitmaps.create(); + final LongBitmap b = LongBitmaps.create(); + for (int j = 0; j < 100; j++) { + a.add(j); + b.add(j + 50); // overlap + } + executor.submit(() -> { + try { + a.or(b); + } catch (Exception e) { + errors.incrementAndGet(); + } finally { + latch.countDown(); + } + }); + executor.submit(() -> { + try { + b.or(a); + } catch (Exception e) { + errors.incrementAndGet(); + } finally { + latch.countDown(); + } + }); + } + + assertTrue(latch.await(30, TimeUnit.SECONDS), + "or() cross-direction should not deadlock"); + executor.shutdown(); + assertEquals(errors.get(), 0); + } + + @Test + public void testDrainToActionDoesNotHoldWriteLock() { + // Verify that the action runs outside the lock: a slow action should not block + // concurrent operations. The select+remove phase holds writeLock briefly, but + // the action invocation happens after unlock. + LongBitmap bitmap = LongBitmaps.create(); + for (int i = 0; i < 1000; i++) { + bitmap.add(i); + } + + // Slow action: sleep briefly per value. drainTo should not hold any lock + // during action execution — concurrent operations should remain responsive. + long start = System.nanoTime(); + long drained = bitmap.drainTo(100, v -> { + try { + Thread.sleep(1); + } catch (InterruptedException e) { + } + }); + long elapsed = System.nanoTime() - start; + assertEquals(drained, 100); + // Total elapsed ~100ms (sleep); the lock was released before action ran. + assertTrue(elapsed >= 100_000_000L); // sanity: drainTo ran the action 100 times + } + + @Test + public void testUint32BoundaryRange() { + // Verify range APIs handle the uint32 upper boundary correctly. + // add(MAX_UINT32, MAX_UINT32+1) should add exactly one value: MAX_UINT32. + // Note: MutableRoaringBitmap.contains(long, long) has a known issue at this + // boundary where it returns false even when the value is present, so we only + // test contains(long) single-value form and cardinality. + LongBitmap bitmap = LongBitmaps.create(); + bitmap.add(0xFFFFFFFFL, 0x100000000L); + assertEquals(bitmap.cardinality(), 1); + assertTrue(bitmap.contains(0xFFFFFFFFL)); + + bitmap.remove(0xFFFFFFFFL, 0x100000000L); + assertEquals(bitmap.cardinality(), 0); + assertFalse(bitmap.contains(0xFFFFFFFFL)); + } + + @Test + public void testRangeReachesMaxUint32WithoutClamp() { + // Validates that add/remove do not need Math.min(to, UINT32_SIZE): + // validateRange(to - 1) already guarantees to <= UINT32_SIZE, and RoaringBitmap + // handles to == UINT32_SIZE correctly at the boundary. If this test ever fails, + // the clamp must be restored. + LongBitmap bitmap = LongBitmaps.create(); + + // Multi-value range that ends exactly at UINT32_SIZE: [MAX-1, MAX+1) = {MAX-1, MAX} + bitmap.add(0xFFFFFFFEL, 0x100000000L); + assertEquals(bitmap.cardinality(), 2); + assertTrue(bitmap.contains(0xFFFFFFFEL)); + assertTrue(bitmap.contains(0xFFFFFFFFL)); + + // Same range on remove + bitmap.remove(0xFFFFFFFEL, 0x100000000L); + assertEquals(bitmap.cardinality(), 0); + assertFalse(bitmap.contains(0xFFFFFFFEL)); + assertFalse(bitmap.contains(0xFFFFFFFFL)); + + // Crossing container boundary (65535/65536) with to on a power-of-2 boundary + bitmap.add(65530L, 65540L); + assertEquals(bitmap.cardinality(), 10); + bitmap.remove(65530L, 65540L); + assertEquals(bitmap.cardinality(), 0); + + // Out-of-range `to` must throw — proving validateRange guards the upper bound. + assertThrows(IllegalArgumentException.class, + () -> bitmap.add(0L, 0x100000001L)); // to = UINT32_SIZE + 1 + } + + @Test + public void testSerializedSizeIsUpperBoundForSerialize() { + // serializedSize() is an upper bound (no runOptimize). serialize() runs + // runOptimize, which only shrinks. So estimated >= actual. + long[] seeds = {0, 100, 65535, 65536, 1L << 30, 0xFFFFFF00L}; + for (int trial = 0; trial < 5; trial++) { + LongBitmap bitmap = LongBitmaps.create(); + java.util.Random rng = new java.util.Random(trial); + int count = 1000 + rng.nextInt(5000); + for (int i = 0; i < count; i++) { + bitmap.add(seeds[rng.nextInt(seeds.length)] + rng.nextInt(100)); + } + long estimated = bitmap.serializedSize(); + byte[] actual = bitmap.serialize(); + assertTrue(estimated >= actual.length, + "trial " + trial + ": estimated " + estimated + " < actual " + actual.length); + } + } + + @Test + public void testDrainToIsAtomic() { + // drainTo must atomically select+remove values. Concurrent add/remove should + // not cause newly added values to be lost between snapshot and removal. + LongBitmap bitmap = LongBitmaps.create(); + for (int i = 0; i < 100; i++) { + bitmap.add(i); + } + + // Drain with a slow action, while concurrently adding/removing values + AtomicInteger errors = new AtomicInteger(0); + ExecutorService executor = Executors.newFixedThreadPool(2); + CountDownLatch latch = new CountDownLatch(2); + + // T1: drain slowly + executor.submit(() -> { + try { + bitmap.drainTo(50, v -> { + try { + Thread.sleep(1); + } catch (InterruptedException e) { + } + }); + } catch (Exception e) { + errors.incrementAndGet(); + } finally { + latch.countDown(); + } + }); + + // T2: add/remove while T1 drains + executor.submit(() -> { + try { + Thread.sleep(10); // let T1 start draining + for (int i = 0; i < 50; i++) { + bitmap.remove(i); + bitmap.add(i); + } + } catch (Exception e) { + errors.incrementAndGet(); + } finally { + latch.countDown(); + } + }); + + try { + assertTrue(latch.await(30, TimeUnit.SECONDS)); + executor.shutdown(); + assertEquals(errors.get(), 0); + // After drain(50) and concurrent add/remove, the bitmap should contain + // the values that were re-added by T2, not lost due to racy andNot. + long finalCount = bitmap.cardinality(); + // We drained ~50, then T2 added back some of those. Final count >= 50 + // (the un-drained values) is the key invariant. + assertTrue(finalCount >= 50, + "finalCount=" + finalCount + " should be >= 50 (un-drained values)"); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } +} From 1e86cf7412a3b4e86c342bc299ec00d7ead6e618 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Thu, 27 Aug 2026 12:48:23 +0800 Subject: [PATCH 194/213] [fix][common] Adapt TripleLongPriorityQueueTest to client compiler release level RandomGenerator.nextLong(long) is a JDK 17+ API and is not visible when pulsar-common is compiled at the client compiler release level. Use nextInt instead; all bounds fit in int and the differential test semantics are unchanged. --- .../util/collections/TripleLongPriorityQueueTest.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueueTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueueTest.java index ee5c671d456cd..737e720894d77 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueueTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueueTest.java @@ -216,9 +216,11 @@ public void testDifferentialRandomPriorityQueue() { boolean doAdd = oracle.isEmpty() || rng.nextBoolean(); if (doAdd) { // ~10% chance of same-prefix (small n1 range) to exercise tie-breaking - long n1 = rng.nextInt(100) < 10 ? rng.nextLong(20) : rng.nextLong(1_000_000); - long n2 = rng.nextLong(100); - long n3 = rng.nextLong(1_000_000); + // note: use nextInt instead of RandomGenerator.nextLong(long) because + // this module is compiled with the client compiler release level + long n1 = rng.nextInt(100) < 10 ? rng.nextInt(20) : rng.nextInt(1_000_000); + long n2 = rng.nextInt(100); + long n3 = rng.nextInt(1_000_000); oracle.add(new long[]{n1, n2, n3}); pq.add(n1, n2, n3); } else { From 889cb3f57964fd01423d3a0302bf91a2311fa1cc Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Fri, 3 Jul 2026 22:51:59 +0800 Subject: [PATCH 195/213] [improve][ml] Replace RangeSetWrapper with PositionRangeSet backed by LongBitmap (#26127) --- conf/broker.conf | 10 +- conf/standalone.conf | 10 +- .../terraform-ansible/templates/broker.conf | 3 - managed-ledger/pom.xml | 8 + .../mledger/ManagedLedgerConfig.java | 15 - .../mledger/impl/ManagedCursorImpl.java | 106 +--- .../mledger/impl/PositionRangeSet.java | 494 +++++++++++++++ .../mledger/impl/RangeSetWrapper.java | 211 ------- .../impl/ManagedCursorConcurrencyTest.java | 2 +- ...edCursorIndividualDeletedMessagesTest.java | 1 - .../mledger/impl/ManagedCursorTest.java | 49 +- .../mledger/impl/ManagedLedgerBkTest.java | 58 +- .../PositionRangeSetCompatibilityTest.java | 198 ++++++ .../mledger/impl/PositionRangeSetTest.java | 580 ++++++++++++++++++ .../mledger/impl/RangeSetWrapperTest.java | 516 ---------------- .../impl/PositionRangeSetBenchmark.java | 136 ++++ .../pulsar/broker/ServiceConfiguration.java | 12 +- .../pulsar/broker/service/BrokerService.java | 2 - .../client/impl/MessageRedeliveryTest.java | 1 - .../collections/ConcurrentRoaringBitmap.java | 109 +++- .../common/util/collections/LongBitmap.java | 69 +++ .../common/util/collections/LongBitmaps.java | 16 + .../util/collections/LongBitmapTest.java | 111 +++- 23 files changed, 1773 insertions(+), 944 deletions(-) create mode 100644 managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionRangeSet.java delete mode 100644 managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/RangeSetWrapper.java create mode 100644 managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetCompatibilityTest.java create mode 100644 managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetTest.java delete mode 100644 managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/RangeSetWrapperTest.java create mode 100644 microbench/src/main/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetBenchmark.java diff --git a/conf/broker.conf b/conf/broker.conf index 7ee46dcd7a399..69721610c7feb 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -1379,14 +1379,8 @@ managedLedgerMaxUnackedRangesToPersist=10000 managedLedgerMaxBatchDeletedIndexToPersist=10000 # When storing acknowledgement state, choose a more compact serialization format that stores -# individual acknowledgements as a bitmap which is serialized to an array of long values. NOTE: This setting requires -# managedLedgerUnackedRangesOpenCacheSetEnabled=true to be effective. -managedLedgerPersistIndividualAckAsLongArray=false - -# When set to true, a BitSet will be used to track acknowledged messages that come after the "mark delete position" -# for each subscription. RoaringBitmap is used as a memory efficient BitSet implementation for the acknowledged -# messages tracking. Unacknowledged ranges are the message ranges excluding the acknowledged messages. -managedLedgerUnackedRangesOpenCacheSetEnabled=true +# individual acknowledgements as a bitmap which is serialized to an array of long values. +managedLedgerPersistIndividualAckAsLongArray=true # Max number of "acknowledgment holes" that can be stored in MetadataStore. If number of unack message range is higher # than this limit then broker will persist unacked ranges into bookkeeper to avoid additional data overhead into diff --git a/conf/standalone.conf b/conf/standalone.conf index 06cd3e38659af..0a50c41d44b85 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -875,14 +875,8 @@ managedLedgerMaxUnackedRangesToPersist=10000 managedLedgerMaxBatchDeletedIndexToPersist=10000 # When storing acknowledgement state, choose a more compact serialization format that stores -# individual acknowledgements as a bitmap which is serialized to an array of long values. NOTE: This setting requires -# managedLedgerUnackedRangesOpenCacheSetEnabled=true to be effective. -managedLedgerPersistIndividualAckAsLongArray=false - -# When set to true, a BitSet will be used to track acknowledged messages that come after the "mark delete position" -# for each subscription. RoaringBitmap is used as a memory efficient BitSet implementation for the acknowledged -# messages tracking. Unacknowledged ranges are the message ranges excluding the acknowledged messages. -managedLedgerUnackedRangesOpenCacheSetEnabled=true +# individual acknowledgements as a bitmap which is serialized to an array of long values. +managedLedgerPersistIndividualAckAsLongArray=true # Max number of "acknowledgment holes" that can be stored in MetadataStore. If number of unack message range is higher # than this limit then broker will persist unacked ranges into bookkeeper to avoid additional data overhead into diff --git a/deployment/terraform-ansible/templates/broker.conf b/deployment/terraform-ansible/templates/broker.conf index 1ccfbacecb04a..51d5bbae0316c 100644 --- a/deployment/terraform-ansible/templates/broker.conf +++ b/deployment/terraform-ansible/templates/broker.conf @@ -1077,9 +1077,6 @@ managedLedgerOffloadMaxThreads=2 # Maximum prefetch rounds for ledger reading for offloading managedLedgerOffloadPrefetchRounds=1 -# Use Open Range-Set to cache unacked messages -managedLedgerUnackedRangesOpenCacheSetEnabled=true - # For Amazon S3 ledger offload, AWS region s3ManagedLedgerOffloadRegion= diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index 3efff29513200..69dad6affc389 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -102,6 +102,14 @@ org.roaringbitmap RoaringBitmap + + it.unimi.dsi + fastutil + + + io.github.merlimat.slog + slog + io.dropwizard.metrics metrics-core diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java index aaa127973a5ec..0eaebaf0f8cfb 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java @@ -33,7 +33,6 @@ import org.apache.bookkeeper.mledger.impl.NullLedgerOffloader; import org.apache.bookkeeper.mledger.intercept.ManagedLedgerInterceptor; import org.apache.commons.collections4.MapUtils; -import org.apache.pulsar.common.util.collections.OpenLongPairRangeSet; /** * Configuration class for a ManagedLedger. @@ -71,7 +70,6 @@ public class ManagedLedgerConfig { private long addEntryTimeoutSeconds = 120; private DigestType digestType = DigestType.CRC32C; private byte[] password = "".getBytes(StandardCharsets.UTF_8); - private boolean unackedRangesOpenCacheSetEnabled = true; private Class bookKeeperEnsemblePlacementPolicyClassName; private Map bookKeeperEnsemblePlacementPolicyProperties; private LedgerOffloader ledgerOffloader = NullLedgerOffloader.INSTANCE; @@ -292,19 +290,6 @@ public ManagedLedgerConfig setPassword(String password) { return this; } - /** - * should use {@link OpenLongPairRangeSet} to store unacked ranges. - * @return - */ - public boolean isUnackedRangesOpenCacheSetEnabled() { - return unackedRangesOpenCacheSetEnabled; - } - - public ManagedLedgerConfig setUnackedRangesOpenCacheSetEnabled(boolean unackedRangesOpenCacheSetEnabled) { - this.unackedRangesOpenCacheSetEnabled = unackedRangesOpenCacheSetEnabled; - return this; - } - /** * @return the metadataEnsemblesize */ diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 49cec68322666..9820f32f85149 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -54,13 +54,13 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; -import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Function; +import java.util.function.IntFunction; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.LongStream; @@ -113,7 +113,6 @@ import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.LongPairRangeSet; import org.apache.pulsar.common.util.collections.LongPairRangeSet.LongPairConsumer; -import org.apache.pulsar.common.util.collections.LongPairRangeSet.RangeBoundConsumer; import org.apache.pulsar.metadata.api.Stat; import org.jspecify.annotations.Nullable; import org.slf4j.Logger; @@ -199,11 +198,8 @@ public class ManagedCursorImpl implements ManagedCursor { private static final LongPairConsumer positionRangeConverter = PositionFactory::create; - private static final RangeBoundConsumer positionRangeReverseConverter = - (position) -> new LongPairRangeSet.LongPair(position.getLedgerId(), position.getEntryId()); - private static final LongPairConsumer recyclePositionRangeConverter = PositionRecyclable::get; - protected final RangeSetWrapper individualDeletedMessages; + protected final PositionRangeSet individualDeletedMessages; // Maintain the deletion status for batch messages // (ledgerId, entryId) -> deletion indexes @@ -366,8 +362,8 @@ protected ManagedCursorImpl(BookKeeper bookkeeper, ManagedLedgerImpl ledger, Str this.cursorProperties = Collections.emptyMap(); this.ledger = ledger; this.name = cursorName; - this.individualDeletedMessages = new RangeSetWrapper<>(positionRangeConverter, - positionRangeReverseConverter, this); + this.individualDeletedMessages = new PositionRangeSet(positionRangeConverter, + getConfig().isPersistentUnackedRangesWithMultipleEntriesEnabled()); if (getConfig().isDeletionAtBatchIndexLevelEnabled()) { this.batchDeletedIndexes = new ConcurrentSkipListMap<>(); } else { @@ -703,22 +699,7 @@ public void recoverIndividualDeletedMessages(PositionInfo positionInfo) { try { Map rangeMap = rangeList.stream().collect(Collectors.toMap(LongListMap::getKey, list -> list.getValuesList().stream().mapToLong(i -> i).toArray())); - // Guarantee compatability for the config "unackedRangesOpenCacheSetEnabled". - if (getConfig().isUnackedRangesOpenCacheSetEnabled()) { - individualDeletedMessages.build(rangeMap); - } else { - RangeSetWrapper rangeSetWrapperV2 = new RangeSetWrapper<>(positionRangeConverter, - positionRangeReverseConverter, true, - getConfig().isPersistentUnackedRangesWithMultipleEntriesEnabled()); - rangeSetWrapperV2.build(rangeMap); - rangeSetWrapperV2.forEach(range -> { - individualDeletedMessages.addOpenClosed(range.lowerEndpoint().getLedgerId(), - range.lowerEndpoint().getEntryId(), range.upperEndpoint().getLedgerId(), - range.upperEndpoint().getEntryId()); - return true; - }); - rangeSetWrapperV2.clear(); - } + individualDeletedMessages.build(rangeMap); } catch (Exception e) { log.warn("[{}]-{} Failed to recover individualDeletedMessages from serialized data", ledger.getName(), name, e); @@ -788,6 +769,15 @@ private void recoverIndividualDeletedMessages(List i } } + @VisibleForTesting + void recoverIndividualDeletedMessages(int count, IntFunction accessor) { + List individualDeletedMessagesList = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + individualDeletedMessagesList.add(accessor.apply(i)); + } + recoverIndividualDeletedMessages(individualDeletedMessagesList); + } + private void recoverBatchDeletedIndexes ( List batchDeletedIndexInfoList) { Objects.requireNonNull(batchDeletedIndexes); @@ -1320,23 +1310,9 @@ public long getEstimatedSizeSinceMarkDeletePosition() { lock.readLock().lock(); try { Range backlogRange = Range.openClosed(markDeletePosition, lastPosition); - - if (getConfig().isUnackedRangesOpenCacheSetEnabled()) { - deletedCount = individualDeletedMessages.cardinality( - backlogRange.lowerEndpoint().getLedgerId(), backlogRange.lowerEndpoint().getEntryId(), - backlogRange.upperEndpoint().getLedgerId(), backlogRange.upperEndpoint().getEntryId()); - } else { - AtomicLong deletedCounter = new AtomicLong(0); - individualDeletedMessages.forEach((r) -> { - if (r.isConnected(backlogRange)) { - Range intersection = r.intersection(backlogRange); - long countInRange = ledger.getNumberOfEntries(intersection); - deletedCounter.addAndGet(countInRange); - } - return true; - }, recyclePositionRangeConverter); - deletedCount = deletedCounter.get(); - } + deletedCount = individualDeletedMessages.cardinality( + backlogRange.lowerEndpoint().getLedgerId(), backlogRange.lowerEndpoint().getEntryId(), + backlogRange.upperEndpoint().getLedgerId(), backlogRange.upperEndpoint().getEntryId()); } finally { lock.readLock().unlock(); } @@ -1894,45 +1870,22 @@ protected long getNumberOfEntries(Range range) { log.debug("[{}] getNumberOfEntries. {} allEntries: {}", ledger.getName(), range, allEntries); } - AtomicLong deletedEntries = new AtomicLong(0); + long deletedEntriesCount = 0; lock.readLock().lock(); try { - if (getConfig().isUnackedRangesOpenCacheSetEnabled()) { - int cardinality = individualDeletedMessages.cardinality( - range.lowerEndpoint().getLedgerId(), range.lowerEndpoint().getEntryId(), - range.upperEndpoint().getLedgerId(), range.upperEndpoint().getEntryId()); - deletedEntries.addAndGet(cardinality); - } else { - individualDeletedMessages.forEach((r) -> { - try { - if (r.isConnected(range)) { - Range commonEntries = r.intersection(range); - long commonCount = ledger.getNumberOfEntries(commonEntries); - if (log.isDebugEnabled()) { - log.debug("[{}] [{}] Discounting {} entries for already deleted range {}", - ledger.getName(), name, commonCount, commonEntries); - } - deletedEntries.addAndGet(commonCount); - } - return true; - } finally { - if (r.lowerEndpoint() instanceof PositionRecyclable) { - ((PositionRecyclable) r.lowerEndpoint()).recycle(); - ((PositionRecyclable) r.upperEndpoint()).recycle(); - } - } - }, recyclePositionRangeConverter); - } + deletedEntriesCount = individualDeletedMessages.cardinality( + range.lowerEndpoint().getLedgerId(), range.lowerEndpoint().getEntryId(), + range.upperEndpoint().getLedgerId(), range.upperEndpoint().getEntryId()); } finally { lock.readLock().unlock(); } if (log.isDebugEnabled()) { - log.debug("[{}] Found {} entries - deleted: {}", ledger.getName(), allEntries - deletedEntries.get(), - deletedEntries); + log.debug("[{}] Found {} entries - deleted: {}", ledger.getName(), allEntries - deletedEntriesCount, + deletedEntriesCount); } - return allEntries - deletedEntries.get(); + return allEntries - deletedEntriesCount; } @@ -3467,16 +3420,7 @@ void persistPositionToLedger(final LedgerHandle lh, MarkDeleteEntry mdEntry, fin .addAllProperties(buildPropertiesMap(mdEntry.properties)); Map internalRanges = null; - /** - * Cursor will create the {@link #individualDeletedMessages} typed {@link LongPairRangeSet.DefaultRangeSet} if - * disabled the config {@link ManagedLedgerConfig#unackedRangesOpenCacheSetEnabled}. - * {@link LongPairRangeSet.DefaultRangeSet} never implemented the methods below: - * - {@link LongPairRangeSet#toRanges(int)}, which is used to serialize cursor metadata. - * - {@link LongPairRangeSet#build(Map)}, which is used to deserialize cursor metadata. - * Do not enable the feature that https://github.com/apache/pulsar/pull/9292 introduced, to avoid serialization - * and deserialization error. - */ - if (getConfig().isUnackedRangesOpenCacheSetEnabled() && getConfig().isPersistIndividualAckAsLongArray()) { + if (getConfig().isPersistIndividualAckAsLongArray()) { lock.readLock().lock(); try { internalRanges = individualDeletedMessages.toRanges(getConfig().getMaxUnackedRangesToPersist()); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionRangeSet.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionRangeSet.java new file mode 100644 index 0000000000000..01e581eb444a1 --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionRangeSet.java @@ -0,0 +1,494 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.impl; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.BoundType; +import com.google.common.collect.Range; +import io.github.merlimat.slog.Logger; +import it.unimi.dsi.fastutil.longs.Long2ObjectRBTreeMap; +import it.unimi.dsi.fastutil.longs.Long2ObjectSortedMap; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.commons.lang3.mutable.MutableInt; +import org.apache.pulsar.common.util.collections.LongBitmap; +import org.apache.pulsar.common.util.collections.LongBitmaps; +import org.apache.pulsar.common.util.collections.LongPairRangeSet; + +/** + * Tracks deleted-message positions as ranges of {@link Position}s. + * + *

The implementation stores positions in a two-level structure: + * the ledger id is used as the map key, and the corresponding entry ids are stored in a + * {@link LongBitmap}. Bit {@code n} in the bitmap of ledger {@code L} represents the position + * {@code (L, n)}. + * + *

Thread Safety

+ * + *

This class is not thread-safe. All methods require the caller to provide external + * synchronization. In normal usage, callers must hold the owning {@link ManagedCursorImpl}'s + * cursor lock before accessing this class. + * + *

The class intentionally does not maintain internal locking. The cursor lock is the single + * synchronization boundary for both this structure and related cursor state. + * + *

Persistence Compatibility

+ * + *

The persisted representation remains compatible with the existing format using + * {@link LongBitmap#serializeToLongArray()} and {@link LongBitmap#deserializeFromLongArray(long[])}, + * which are compatible with the BitSet long[] format. + * + *

Entry ids are stored as bitmap indexes and therefore use {@code int} values. This is safe + * because {@code managedLedgerMaxEntriesPerLedger} is an {@code int}, so valid entry ids are within + * {@code [0, Integer.MAX_VALUE]}. + */ +class PositionRangeSet implements LongPairRangeSet { + + private static final Logger log = Logger.get(PositionRangeSet.class); + + private static final long EARLIEST_LEDGER_ID = -1L; + private static final long EARLIEST_ENTRY_ID = -1L; + private static final long LATEST_LEDGER_ID = Long.MAX_VALUE; + private static final long LATEST_ENTRY_ID = Long.MAX_VALUE; + + /** + * Maps ledger ID to a bitmap of deleted entry IDs within that ledger. + * Bit {@code n} in the bitmap represents entry {@code n} in the ledger. + */ + private final Long2ObjectSortedMap rangeBitmapMap = new Long2ObjectRBTreeMap<>(); + private final LongPairConsumer consumer; + private final boolean enableMultiEntry; + + private final LongBitmap dirtyLedgers = LongBitmaps.create(); + + private int cachedSize = 0; + private String cachedToString = "[]"; + private boolean updatedAfterCachedForSize = true; + private boolean updatedAfterCachedForToString = true; + + PositionRangeSet(LongPairConsumer consumer, boolean enableMultiEntry) { + this.consumer = consumer; + this.enableMultiEntry = enableMultiEntry; + } + + private static long lastPresentValue(LongBitmap bitmap) { + return bitmap.lastPresentValue(); + } + + @Override + public void addOpenClosed(long lowerLedgerId, long lowerEntryIdOpen, long upperLedgerId, long upperEntryId) { + if (enableMultiEntry) { + markDirty(lowerLedgerId, upperLedgerId); + } + long lowerEntryId = lowerEntryIdOpen + 1; + if (lowerLedgerId != upperLedgerId) { + // Extend lower ledger's bitmap only if it already exists and has bits at/after lowerEntryId; + // otherwise we'd invent acknowledgements that never happened (e.g. (2:10..4:10] must not + // touch 2:10 if ledger 2 was empty). + if (isValid(lowerLedgerId, lowerEntryId)) { + LongBitmap rangeBitmap = rangeBitmapMap.get(lowerLedgerId); + if (rangeBitmap != null) { + long lastEntryId = rangeBitmap.lastPresentValue(); + if (lastEntryId > lowerEntryIdOpen) { + rangeBitmap.add(lowerEntryId, Math.max(lastEntryId, lowerEntryId) + 1); + } + } + } + if (isValid(upperLedgerId, upperEntryId)) { + LongBitmap rangeBitmap = rangeBitmapMap.computeIfAbsent(upperLedgerId, k -> LongBitmaps.create()); + rangeBitmap.add(0, upperEntryId + 1); + } + } else { + LongBitmap rangeBitmap = rangeBitmapMap.computeIfAbsent(lowerLedgerId, k -> LongBitmaps.create()); + rangeBitmap.add(lowerEntryId, upperEntryId + 1); + } + invalidateCaches(); + } + + @Override + public boolean contains(long ledgerId, long entryId) { + LongBitmap rangeBitmap = rangeBitmapMap.get(ledgerId); + if (rangeBitmap != null) { + return rangeBitmap.contains(getSafeEntry(entryId)); + } + return false; + } + + @Override + public Range rangeContaining(long ledgerId, long entryId) { + LongBitmap rangeBitmap = rangeBitmapMap.get(ledgerId); + if (rangeBitmap == null || !rangeBitmap.contains(getSafeEntry(entryId))) { + return null; + } + long safeEntryId = getSafeEntry(entryId); + long lowerEntryId = rangeBitmap.previousAbsentValue(safeEntryId) + 1; + Position lower = consumer.apply(ledgerId, lowerEntryId); + long nextAbsentEntryId = rangeBitmap.nextAbsentValue(safeEntryId); + Position upper = consumer.apply(ledgerId, Math.max(nextAbsentEntryId - 1, lowerEntryId)); + return Range.closed(lower, upper); + } + + @Override + public void removeAtMost(long ledgerId, long entryId) { + if (enableMultiEntry && ledgerId >= 0) { + long end = Math.min(ledgerId + 1L, (long) Integer.MAX_VALUE + 1); + dirtyLedgers.remove(0, end); + } + remove(Range.atMost(PositionFactory.create(ledgerId, entryId))); + } + + @Override + public boolean isEmpty() { + if (rangeBitmapMap.isEmpty()) { + return true; + } + for (LongBitmap bitmap : rangeBitmapMap.values()) { + if (!bitmap.isEmpty()) { + return false; + } + } + return true; + } + + @Override + public void clear() { + rangeBitmapMap.clear(); + resetDirtyKeys(); + invalidateCaches(); + } + + @Override + public Range span() { + if (rangeBitmapMap.isEmpty()) { + return null; + } + long firstLedgerId = rangeBitmapMap.firstLongKey(); + long lastLedgerId = rangeBitmapMap.lastLongKey(); + LongBitmap firstBitmap = rangeBitmapMap.get(firstLedgerId); + LongBitmap lastBitmap = rangeBitmapMap.get(lastLedgerId); + long firstEntryId = firstBitmap.nextPresentValue(0); + long lastEntryId = lastBitmap.lastPresentValue(); + return Range.openClosed(consumer.apply(firstLedgerId, firstEntryId - 1), + consumer.apply(lastLedgerId, lastEntryId)); + } + + @Override + public List> asRanges() { + List> ranges = new ArrayList<>(); + forEach(range -> { + ranges.add(range); + return true; + }); + return ranges; + } + + @Override + public void forEach(RangeProcessor action) { + forEach(action, consumer); + } + + @Override + public void forEach(RangeProcessor action, LongPairConsumer consumerParam) { + forEachRawRange((lowerLedgerId, lowerEntryId, upperLedgerId, upperEntryId) -> { + Range range = Range.openClosed( + consumerParam.apply(lowerLedgerId, lowerEntryId), + consumerParam.apply(upperLedgerId, upperEntryId)); + return action.process(range); + }); + } + + @Override + public void forEachRawRange(RawRangeProcessor processor) { + AtomicBoolean completed = new AtomicBoolean(false); + rangeBitmapMap.forEach((ledgerId, bitmap) -> { + if (completed.get() || bitmap.isEmpty()) { + return; + } + long firstEntryId = bitmap.nextPresentValue(0); + long lastEntryId = bitmap.lastPresentValue(); + long currentEntryId = firstEntryId; + while (currentEntryId != -1 && currentEntryId <= lastEntryId) { + long nextAbsentEntryId = bitmap.nextAbsentValue(currentEntryId); + if (!processor.processRawRange(ledgerId, currentEntryId - 1, ledgerId, nextAbsentEntryId - 1)) { + completed.set(true); + break; + } + if (nextAbsentEntryId > Integer.MAX_VALUE) { + break; + } + currentEntryId = bitmap.nextPresentValue(nextAbsentEntryId); + } + }); + } + + @Override + public Range firstRange() { + if (rangeBitmapMap.isEmpty()) { + return null; + } + long firstLedgerId = rangeBitmapMap.firstLongKey(); + LongBitmap firstBitmap = rangeBitmapMap.get(firstLedgerId); + long lowerEntryId = firstBitmap.nextPresentValue(0); + long upperEntryId = Math.max(lowerEntryId, firstBitmap.nextAbsentValue(lowerEntryId) - 1); + return Range.openClosed(consumer.apply(firstLedgerId, lowerEntryId - 1), + consumer.apply(firstLedgerId, upperEntryId)); + } + + @Override + public Range lastRange() { + if (rangeBitmapMap.isEmpty()) { + return null; + } + long lastLedgerId = rangeBitmapMap.lastLongKey(); + LongBitmap lastBitmap = rangeBitmapMap.get(lastLedgerId); + long upperEntryId = lastBitmap.lastPresentValue(); + long lowerEntryId = Math.min(lastBitmap.previousAbsentValue(upperEntryId), upperEntryId); + return Range.openClosed(consumer.apply(lastLedgerId, lowerEntryId), + consumer.apply(lastLedgerId, upperEntryId)); + } + + @Override + public Map toRanges(int maxRanges) { + Map internalBitSetMap = new HashMap<>(); + MutableInt rangeCount = new MutableInt(); + rangeBitmapMap.forEach((ledgerId, bitmap) -> { + if (rangeCount.addAndGet((int) bitmap.cardinality()) > maxRanges) { + return; + } + internalBitSetMap.put(ledgerId, bitmap.serializeToLongArray()); + }); + return internalBitSetMap; + } + + @Override + public void build(Map internalRange) { + rangeBitmapMap.clear(); + resetDirtyKeys(); + + internalRange.forEach((ledgerId, ranges) -> { + rangeBitmapMap.put(ledgerId.longValue(), LongBitmaps.deserializeFromLongArray(ranges)); + }); + invalidateCaches(); + } + + @Override + public int cardinality(long lowerLedgerId, long lowerEntryId, long upperLedgerId, long upperEntryId) { + Long2ObjectSortedMap subMap = rangeBitmapMap.subMap(lowerLedgerId, upperLedgerId + 1); + MutableInt v = new MutableInt(0); + subMap.forEach((ledgerId, bitmap) -> { + if (ledgerId == lowerLedgerId && ledgerId == upperLedgerId) { + long count = bitmap.rank(upperEntryId + 1) - bitmap.rank(lowerEntryId); + v.add(Math.toIntExact(count)); + } else if (ledgerId == lowerLedgerId) { + long count = bitmap.cardinality() - bitmap.rank(lowerEntryId); + v.add(Math.toIntExact(count)); + } else if (ledgerId == upperLedgerId) { + long count = bitmap.rank(upperEntryId + 1); + v.add(Math.toIntExact(count)); + } else { + v.add(Math.toIntExact(bitmap.cardinality())); + } + }); + return v.intValue(); + } + + @Override + public int size() { + if (updatedAfterCachedForSize) { + MutableInt size = new MutableInt(0); + forEachRawRange((lowerLedgerId, lowerEntryId, upperLedgerId, upperEntryId) -> { + size.increment(); + return true; + }); + cachedSize = size.intValue(); + updatedAfterCachedForSize = false; + } + return cachedSize; + } + + @Override + public int hashCode() { + return Objects.hashCode(rangeBitmapMap); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof PositionRangeSet other)) { + return false; + } + if (this == obj) { + return true; + } + return this.rangeBitmapMap.equals(other.rangeBitmapMap); + } + + @Override + public String toString() { + if (updatedAfterCachedForToString) { + StringBuilder sb = new StringBuilder(); + sb.append("["); + AtomicBoolean first = new AtomicBoolean(true); + forEach(range -> { + if (!first.get()) { + sb.append(","); + } + sb.append(range); + first.set(false); + return true; + }); + sb.append("]"); + cachedToString = sb.toString(); + updatedAfterCachedForToString = false; + } + return cachedToString; + } + + @VisibleForTesting + void add(Range range) { + Position lowerEndpoint = range.hasLowerBound() ? range.lowerEndpoint() + : PositionFactory.create(EARLIEST_LEDGER_ID, EARLIEST_ENTRY_ID); + Position upperEndpoint = range.hasUpperBound() ? range.upperEndpoint() + : PositionFactory.create(LATEST_LEDGER_ID, LATEST_ENTRY_ID); + + long lowerEntryIdOpen = (range.hasLowerBound() && range.lowerBoundType().equals(BoundType.CLOSED)) + ? getSafeEntry(lowerEndpoint) - 1 + : getSafeEntry(lowerEndpoint); + long upperEntryIdClosed = (range.hasUpperBound() && range.upperBoundType().equals(BoundType.CLOSED)) + ? getSafeEntry(upperEndpoint) + : getSafeEntry(upperEndpoint) + 1; + + rangeBitmapMap.computeIfAbsent(lowerEndpoint.getLedgerId(), k -> LongBitmaps.create()) + .add(lowerEntryIdOpen + 1); + addOpenClosed(lowerEndpoint.getLedgerId(), lowerEntryIdOpen, + upperEndpoint.getLedgerId(), upperEntryIdClosed); + } + + @VisibleForTesting + void remove(Range range) { + Position lowerEndpoint = range.hasLowerBound() ? range.lowerEndpoint() + : PositionFactory.create(EARLIEST_LEDGER_ID, EARLIEST_ENTRY_ID); + Position upperEndpoint = range.hasUpperBound() ? range.upperEndpoint() + : PositionFactory.create(LATEST_LEDGER_ID, LATEST_ENTRY_ID); + + long lowerEntryId = (range.hasLowerBound() && range.lowerBoundType().equals(BoundType.CLOSED)) + ? getSafeEntry(lowerEndpoint) + : getSafeEntry(lowerEndpoint) + 1; + long upperEntryId = (range.hasUpperBound() && range.upperBoundType().equals(BoundType.CLOSED)) + ? getSafeEntry(upperEndpoint) + : getSafeEntry(upperEndpoint) - 1; + + long lowerLedgerId = lowerEndpoint.getLedgerId(); + long upperLedgerId = upperEndpoint.getLedgerId(); + boolean lowerIsEarliest = lowerLedgerId == EARLIEST_LEDGER_ID + && lowerEndpoint.getEntryId() == EARLIEST_ENTRY_ID; + boolean upperIsLatest = upperLedgerId == LATEST_LEDGER_ID + && upperEndpoint.getEntryId() == LATEST_ENTRY_ID; + boolean sameLedger = lowerLedgerId == upperLedgerId; + + if (lowerIsEarliest) { + rangeBitmapMap.headMap(upperLedgerId).clear(); + } + if (upperIsLatest) { + rangeBitmapMap.tailMap(lowerLedgerId + 1).clear(); + } + if (!sameLedger && !lowerIsEarliest && !upperIsLatest) { + rangeBitmapMap.subMap(lowerLedgerId + 1, upperLedgerId).clear(); + } + + LongBitmap lowerSet = lowerIsEarliest ? null : rangeBitmapMap.get(lowerLedgerId); + LongBitmap upperSet = upperIsLatest ? null + : (sameLedger ? lowerSet : rangeBitmapMap.get(upperLedgerId)); + + if (sameLedger && lowerSet != null) { + lowerSet.remove(lowerEntryId, upperEntryId + 1); + } else { + if (lowerSet != null) { + lowerSet.remove(lowerEntryId, lastPresentValue(lowerSet)); + } + if (upperSet != null) { + upperSet.remove(0, upperEntryId + 1); + } + } + + if (lowerSet != null && lowerSet.isEmpty()) { + rangeBitmapMap.remove(lowerLedgerId); + } + if (!sameLedger && upperSet != null && upperSet.isEmpty()) { + rangeBitmapMap.remove(upperLedgerId); + } + + invalidateCaches(); + } + + void resetDirtyKeys() { + dirtyLedgers.clear(); + } + + boolean isDirtyLedgers(long ledgerId) { + return ledgerId >= 0 && ledgerId <= Integer.MAX_VALUE && dirtyLedgers.contains(ledgerId); + } + + private void markDirty(long lowerLedgerId, long upperLedgerId) { + // Original semantics: dirtyLedgers.addOpenClosed(k1, 0, k2, 0), which in LongPair ordering + // is (k1, k2] on ledger ids. LongBitmap.add(from, to) is half-open [from, to), so shift both + // bounds. Same-ledger or inverted range is a no-op. + // + // Note: Ledger IDs are 64-bit longs, but LongBitmap supports unsigned 32-bit range [0, 2^32-1]. + // In practice, BookKeeper ledger IDs rarely exceed Integer.MAX_VALUE. If upperLedgerId exceeds + // this limit, we skip tracking to avoid overflow. This is acceptable because: + // 1. The dirty tracker is an optimization hint for selective persistence + // 2. Missing a dirty mark means conservative full-ledger write (safe, just slower) + // 3. Real-world ledger IDs stay well within 32-bit range + if (upperLedgerId <= lowerLedgerId || lowerLedgerId < 0) { + return; + } + if (lowerLedgerId >= Integer.MAX_VALUE || upperLedgerId > Integer.MAX_VALUE) { + log.warn() + .attr("lowerLedgerId", lowerLedgerId) + .attr("upperLedgerId", upperLedgerId) + .log("Skipping dirty tracking for ledger ID at/exceeding Integer.MAX_VALUE"); + return; + } + dirtyLedgers.add(lowerLedgerId + 1, upperLedgerId + 1); + } + + private boolean isValid(long ledgerId, long entryId) { + return ledgerId != EARLIEST_LEDGER_ID && entryId != EARLIEST_ENTRY_ID + && ledgerId != LATEST_LEDGER_ID && entryId != LATEST_ENTRY_ID; + } + + private int getSafeEntry(Position position) { + return getSafeEntry(position.getEntryId()); + } + + private int getSafeEntry(long value) { + return (int) Math.max(value, -1); + } + + private void invalidateCaches() { + updatedAfterCachedForSize = true; + updatedAfterCachedForToString = true; + } +} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/RangeSetWrapper.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/RangeSetWrapper.java deleted file mode 100644 index 76ac3e1be726c..0000000000000 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/RangeSetWrapper.java +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.bookkeeper.mledger.impl; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Range; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import org.apache.pulsar.common.util.collections.LongPairRangeSet; -import org.apache.pulsar.common.util.collections.OpenLongPairRangeSet; -import org.roaringbitmap.RoaringBitSet; - -/** - * Wraps other Range classes, and adds LRU, marking dirty data and other features on this basis. - * This range set is not thread safety. - * - * @param - */ -public class RangeSetWrapper> implements LongPairRangeSet { - - private final LongPairRangeSet rangeSet; - private final LongPairConsumer rangeConverter; - private final boolean enableMultiEntry; - - /** - * Record which Ledger is dirty. - */ - private final DefaultRangeSet dirtyLedgers = new LongPairRangeSet.DefaultRangeSet<>( - (LongPairConsumer) (key, value) -> key, - (RangeBoundConsumer) key -> new LongPair(key, 0)); - - public RangeSetWrapper(LongPairConsumer rangeConverter, - RangeBoundConsumer rangeBoundConsumer, - ManagedCursorImpl managedCursor) { - this(rangeConverter, rangeBoundConsumer, managedCursor.getConfig().isUnackedRangesOpenCacheSetEnabled(), - managedCursor.getConfig().isPersistentUnackedRangesWithMultipleEntriesEnabled()); - } - - public RangeSetWrapper(LongPairConsumer rangeConverter, - RangeBoundConsumer rangeBoundConsumer, - boolean unackedRangesOpenCacheSetEnabled, - boolean persistentUnackedRangesWithMultipleEntriesEnabled) { - this.rangeConverter = rangeConverter; - this.rangeSet = unackedRangesOpenCacheSetEnabled - ? new OpenLongPairRangeSet<>(rangeConverter, RoaringBitSet::new) - : new LongPairRangeSet.DefaultRangeSet<>(rangeConverter, rangeBoundConsumer); - this.enableMultiEntry = persistentUnackedRangesWithMultipleEntriesEnabled; - } - - @Override - public void addOpenClosed(long lowerKey, long lowerValue, long upperKey, long upperValue) { - if (enableMultiEntry) { - dirtyLedgers.addOpenClosed(lowerKey, 0, upperKey, 0); - } - rangeSet.addOpenClosed(lowerKey, lowerValue, upperKey, upperValue); - } - - @Override - public boolean contains(long key, long value) { - return rangeSet.contains(key, value); - } - - @Override - public Range rangeContaining(long key, long value) { - return rangeSet.rangeContaining(key, value); - } - - @Override - public void removeAtMost(long key, long value) { - if (enableMultiEntry) { - dirtyLedgers.removeAtMost(key, 0); - } - rangeSet.removeAtMost(key, value); - } - - @Override - public boolean isEmpty() { - return rangeSet.isEmpty(); - } - - @Override - public void clear() { - rangeSet.clear(); - dirtyLedgers.clear(); - } - - @Override - public Range span() { - return rangeSet.span(); - } - - @Override - public Collection> asRanges() { - Collection> collection = rangeSet.asRanges(); - if (collection instanceof List) { - return collection; - } - return new ArrayList<>(collection); - } - - @Override - public void forEach(RangeProcessor action) { - rangeSet.forEach(action); - } - - @Override - public void forEach(RangeProcessor action, LongPairConsumer consumer) { - rangeSet.forEach(action, consumer); - } - - @Override - public void forEachRawRange(RawRangeProcessor action) { - rangeSet.forEachRawRange(action); - } - - @Override - public int size() { - return rangeSet.size(); - } - - @Override - public Range firstRange() { - return rangeSet.firstRange(); - } - - @Override - public Range lastRange() { - return rangeSet.lastRange(); - } - - @Override - public Map toRanges(int maxRanges) { - return rangeSet.toRanges(maxRanges); - } - - @Override - public void build(Map internalRange) { - rangeSet.build(internalRange); - } - - @Override - public int cardinality(long lowerKey, long lowerValue, long upperKey, long upperValue) { - return rangeSet.cardinality(lowerKey, lowerValue, upperKey, upperValue); - } - - @VisibleForTesting - void add(Range range) { - if (!(rangeSet instanceof OpenLongPairRangeSet)) { - throw new UnsupportedOperationException("Only ConcurrentOpenLongPairRangeSet support this method"); - } - ((OpenLongPairRangeSet) rangeSet).add(range); - } - - @VisibleForTesting - void remove(Range range) { - if (rangeSet instanceof OpenLongPairRangeSet) { - ((OpenLongPairRangeSet) rangeSet).remove((Range) range); - } else { - ((DefaultRangeSet) rangeSet).remove(range); - } - } - - public void resetDirtyKeys() { - dirtyLedgers.clear(); - } - - public boolean isDirtyLedgers(long ledgerId) { - return dirtyLedgers.contains(ledgerId); - } - - @Override - public String toString() { - return rangeSet.toString(); - } - - @Override - public int hashCode() { - return rangeSet.hashCode(); - } - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof RangeSetWrapper)) { - return false; - } - if (this == obj) { - return true; - } - @SuppressWarnings("rawtypes") - RangeSetWrapper set = (RangeSetWrapper) obj; - return this.rangeSet.equals(set.rangeSet); - } -} diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java index f7b6e755bce83..1b8c999eef37a 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java @@ -70,7 +70,7 @@ public void deleteFailed(ManagedLedgerException exception, Object ctx) { @Test(dataProvider = "useOpenRangeSet") public void testMarkDeleteAndRead(boolean useOpenRangeSet) throws Exception { ManagedLedgerConfig config = new ManagedLedgerConfig().setMaxEntriesPerLedger(2) - .setUnackedRangesOpenCacheSetEnabled(useOpenRangeSet); + ; ManagedLedger ledger = factory.open("my_test_ledger", config); final ManagedCursor cursor = ledger.openCursor("c1"); diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorIndividualDeletedMessagesTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorIndividualDeletedMessagesTest.java index b6c4cc5895db5..86e3bcbd22307 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorIndividualDeletedMessagesTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorIndividualDeletedMessagesTest.java @@ -44,7 +44,6 @@ void testRecoverIndividualDeletedMessages() throws Exception { BookKeeper bookkeeper = mock(BookKeeper.class); ManagedLedgerConfig config = new ManagedLedgerConfig(); - config.setUnackedRangesOpenCacheSetEnabled(true); NavigableMap ledgersInfo = new ConcurrentSkipListMap<>(); ledgersInfo.put(1L, createLedgerInfo(1, 100, 1024)); diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java index 1dcff5762802a..631437f89d9e9 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java @@ -275,7 +275,7 @@ public void testConcurrentPropertyOperationsThreadSafety() throws Exception { } executor.shutdown(); - // Wait for each task to complete with timeout + // Wait for each task to complete with timeou for (Future future : allFutures) { try { future.get(30, TimeUnit.SECONDS); @@ -297,7 +297,7 @@ public void testConcurrentPropertyOperationsThreadSafety() throws Exception { // 2. No inconsistent states detected assertFalse(inconsistencyDetected.get(), "No inconsistent states (key with null value) should be detected"); - // 3: Final cursor state should be internally consistent + // 3: Final cursor state should be internally consisten Map finalProperties = cursor.getProperties(); try { for (Map.Entry entry : finalProperties.entrySet()) { @@ -985,7 +985,7 @@ void testResetCursor1() throws Exception { final AtomicBoolean moveStatus = new AtomicBoolean(false); - // reset to earliest + // reset to earlies Position earliest = PositionFactory.EARLIEST; try { cursor.resetCursor(earliest); @@ -1555,7 +1555,7 @@ public void asyncMarkDeleteBlockingWithOneShot() throws Exception { // just for log debug purpose Deque positions = new ConcurrentLinkedDeque<>(); - // In previous flaky test, we set num=100, PR https://github.com/apache/pulsar/pull/25087 will make the test + // In previous flaky test, we set num=100, PR https://github.com/apache/pulsar/pull/25087 will make the tes // more flaky. Flaky case: // 1. cursor recovered with markDeletePosition 12:9, persistentMarkDeletePosition 12:9. // 2. cursor recovered with mark markDeletePosition 13:-1, persistentMarkDeletePosition 13:-1. @@ -2079,7 +2079,7 @@ void testCountingWithDeletedEntries() throws Exception { @Test(timeOut = 20000, dataProvider = "useOpenRangeSet") void testMarkDeleteTwice(boolean useOpenRangeSet) throws Exception { ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig() - .setUnackedRangesOpenCacheSetEnabled(useOpenRangeSet).setMaxEntriesPerLedger(2)); + .setMaxEntriesPerLedger(2)); ManagedCursor cursor = ledger.openCursor("c1"); Position p1 = ledger.addEntry("entry1".getBytes()); @@ -2092,7 +2092,7 @@ void testMarkDeleteTwice(boolean useOpenRangeSet) throws Exception { @Test(timeOut = 20000, dataProvider = "useOpenRangeSet") void testSkipEntries(boolean useOpenRangeSet) throws Exception { ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger", new ManagedLedgerConfig() - .setUnackedRangesOpenCacheSetEnabled(useOpenRangeSet).setMaxEntriesPerLedger(2)); + .setMaxEntriesPerLedger(2)); Position pos; ManagedCursor c1 = ledger.openCursor("c1"); @@ -2142,7 +2142,7 @@ void testSkipEntries(boolean useOpenRangeSet) throws Exception { @Test(timeOut = 20000, dataProvider = "useOpenRangeSet") void testSkipEntriesWithIndividualDeletedMessages(boolean useOpenRangeSet) throws Exception { ManagedLedger ledger = factory.open("testSkipEntriesWithIndividualDeletedMessages", new ManagedLedgerConfig() - .setUnackedRangesOpenCacheSetEnabled(useOpenRangeSet).setMaxEntriesPerLedger(5)); + .setMaxEntriesPerLedger(5)); ManagedCursor c1 = ledger.openCursor("c1"); Position pos1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding)); @@ -2190,7 +2190,7 @@ void testSkipEntriesWithIndividualDeletedMessages(boolean useOpenRangeSet) throw @Test(timeOut = 20000, dataProvider = "useOpenRangeSet") void testClearBacklog(boolean useOpenRangeSet) throws Exception { ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig() - .setUnackedRangesOpenCacheSetEnabled(useOpenRangeSet).setMaxEntriesPerLedger(1)); + .setMaxEntriesPerLedger(1)); ManagedCursor c1 = ledger.openCursor("c1"); ledger.addEntry("dummy-entry-1".getBytes(Encoding)); @@ -2242,7 +2242,7 @@ void testClearBacklog(boolean useOpenRangeSet) throws Exception { @Test(timeOut = 20000, dataProvider = "useOpenRangeSet") void testRateLimitMarkDelete(boolean useOpenRangeSet) throws Exception { ManagedLedgerConfig config = new ManagedLedgerConfig(); - config.setThrottleMarkDelete(1).setUnackedRangesOpenCacheSetEnabled(useOpenRangeSet); // Throttle to 1/s + config.setThrottleMarkDelete(1); // Throttle to 1/s ManagedLedger ledger = factory.open("my_test_ledger", config); ManagedCursor c1 = ledger.openCursor("c1"); @@ -2271,7 +2271,7 @@ void testRateLimitMarkDelete(boolean useOpenRangeSet) throws Exception { @Test(timeOut = 20000, dataProvider = "useOpenRangeSet") void deleteSingleMessageTwice(boolean useOpenRangeSet) throws Exception { ManagedLedger ledger = factory.open("my_test_ledger", - new ManagedLedgerConfig().setUnackedRangesOpenCacheSetEnabled(useOpenRangeSet)); + new ManagedLedgerConfig()); ManagedCursor c1 = ledger.openCursor("c1"); @@ -2335,11 +2335,10 @@ void deleteSingleMessageTwice(boolean useOpenRangeSet) throws Exception { assertEquals(c1.getMarkDeletedPosition(), p4); assertEquals(c1.getReadPosition(), p4.getNext()); } - @Test(timeOut = 10000, dataProvider = "useOpenRangeSet") void testReadEntriesOrWait(boolean useOpenRangeSet) throws Exception { ManagedLedger ledger = factory.open("my_test_ledger", - new ManagedLedgerConfig().setUnackedRangesOpenCacheSetEnabled(useOpenRangeSet)); + new ManagedLedgerConfig()); final int consumers = 10; final CountDownLatch counter = new CountDownLatch(consumers); @@ -2485,7 +2484,7 @@ void testScan(int numEntries, int batchSize) throws Exception { return true; }), batchSize, 1, Long.MAX_VALUE).get()); - // timeout + // timeou // please note that the timeout is verified // between the reads // so with a big batchSize this test would take too much @@ -2754,7 +2753,7 @@ void testFindNewestMatchingEdgeCase5() throws Exception { @Test(timeOut = 20000, dataProvider = "useOpenRangeSet") void testFindNewestMatchingEdgeCase6(boolean useOpenRangeSet) throws Exception { ManagedLedger ledger = factory.open("testFindNewestMatchingEdgeCase6", new ManagedLedgerConfig() - .setUnackedRangesOpenCacheSetEnabled(useOpenRangeSet).setMaxEntriesPerLedger(3)); + .setMaxEntriesPerLedger(3)); ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1"); ledger.addEntry("expired".getBytes(Encoding)); @@ -2773,7 +2772,7 @@ void testFindNewestMatchingEdgeCase6(boolean useOpenRangeSet) throws Exception { @Test(timeOut = 20000, dataProvider = "useOpenRangeSet") void testFindNewestMatchingEdgeCase7(boolean useOpenRangeSet) throws Exception { ManagedLedger ledger = factory.open("testFindNewestMatchingEdgeCase7", - new ManagedLedgerConfig().setUnackedRangesOpenCacheSetEnabled(useOpenRangeSet)); + new ManagedLedgerConfig()); ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1"); ledger.addEntry("expired".getBytes(Encoding)); @@ -2868,7 +2867,7 @@ void testFindNewestMatchingEdgeCase10() throws Exception { @Test(timeOut = 20000, dataProvider = "useOpenRangeSet") void testIndividuallyDeletedMessages(boolean useOpenRangeSet) throws Exception { ManagedLedger ledger = factory.open("testIndividuallyDeletedMessages", - new ManagedLedgerConfig().setUnackedRangesOpenCacheSetEnabled(useOpenRangeSet)); + new ManagedLedgerConfig()); ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1"); ledger.addEntry("entry-0".getBytes(Encoding)); @@ -2908,7 +2907,7 @@ void testIndividuallyDeletedMessages1() throws Exception { @Test(timeOut = 20000, dataProvider = "useOpenRangeSet") void testIndividuallyDeletedMessages2(boolean useOpenRangeSet) throws Exception { ManagedLedger ledger = factory.open("testIndividuallyDeletedMessages2", - new ManagedLedgerConfig().setUnackedRangesOpenCacheSetEnabled(useOpenRangeSet)); + new ManagedLedgerConfig()); ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1"); ledger.addEntry("entry-0".getBytes(Encoding)); @@ -2929,7 +2928,7 @@ void testIndividuallyDeletedMessages2(boolean useOpenRangeSet) throws Exception @Test(timeOut = 20000, dataProvider = "useOpenRangeSet") void testIndividuallyDeletedMessages3(boolean useOpenRangeSet) throws Exception { ManagedLedger ledger = factory.open("testIndividuallyDeletedMessages3", - new ManagedLedgerConfig().setUnackedRangesOpenCacheSetEnabled(useOpenRangeSet)); + new ManagedLedgerConfig()); ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1"); ledger.addEntry("entry-0".getBytes(Encoding)); @@ -2973,7 +2972,7 @@ void testFindNewestMatchingAfterLedgerRollover() throws Exception { // then we are done // there was a bug (https://github.com/apache/pulsar/issues/9082) // in which if the last message was in a different ledger - // the jump from the first message to the last message went + // the jump from the first message to the last message wen // to an invalid position and so the search stopped at the first message // we want to assert here that the algorithm returns the position of the @@ -3116,7 +3115,7 @@ void testReplayEntries() throws Exception { ledger.addEntry("entry4".getBytes(Encoding)); // 1. Replay empty position set should return empty entry set - Set positions = new HashSet(); + Set positions = new HashSet<>(); assertTrue(c1.replayEntries(positions).isEmpty()); positions.add(p1); @@ -3539,7 +3538,7 @@ public void operationFailed(ManagedLedgerException exception) { String path = "/managed-ledgers/my_test_ledger/c1"; metadataStore.put(path, "".getBytes(), Optional.empty()).join(); - // try to create ledger again which will fail because managedCursorInfo znode is already updated with different + // try to create ledger again which will fail because managedCursorInfo znode is already updated with differen // version so, this call will fail with BadVersionException CountDownLatch latch2 = new CountDownLatch(1); // create ledger will create ledgerId = 6 @@ -3979,7 +3978,7 @@ public void deleteFailed(ManagedLedgerException exception, Object ctx) { assertEquals(actualBacklogSize, expectedBacklogSize, "Backlog size should account for individual deletions"); - // Verify both count and size are correct + // Verify both count and size are correc assertEquals(cursor.getNumberOfEntriesInBacklog(true), 2, "Backlog count should be 2"); ledger.close(); @@ -4817,7 +4816,7 @@ public void testCursorNoRolloverIfNoMetadataSession() throws Exception { assertEquals(cursor.getCursorLedger(), initialLedgerId); - // After the session gets reestablished, the rollover should restart + // After the session gets reestablished, the rollover should restar metadataStore.triggerSessionEvent(SessionEvent.SessionReestablished); for (int i = 0; i < 10; i++) { @@ -5454,7 +5453,7 @@ public void findEntryFailed(ManagedLedgerException exception, assertNotNull(positionRef.get()); assertEquals(positionRef.get(), position1); - // find the newest entry with start + // find the newest entry with star AtomicBoolean failed1 = new AtomicBoolean(false); CountDownLatch latch1 = new CountDownLatch(1); AtomicReference positionRef1 = new AtomicReference<>(); @@ -5611,7 +5610,7 @@ public void findEntryFailed(ManagedLedgerException exception, assertNotNull(positionRef.get()); assertEquals(positionRef.get(), position3); - // find the newest entry with start + // find the newest entry with star AtomicBoolean failed1 = new AtomicBoolean(false); CountDownLatch latch1 = new CountDownLatch(1); AtomicReference positionRef1 = new AtomicReference<>(); diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java index 1ba3c0f1ed332..71c38bbe64ca3 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java @@ -704,62 +704,6 @@ public void testPeriodicRollover() throws Exception { Awaitility.await().until(() -> cursorImpl.getCursorLedger() != currentLedgerId); } - @DataProvider(name = "unackedRangesOpenCacheSetEnabledPair") - public Object[][] unackedRangesOpenCacheSetEnabledPair() { - return new Object[][]{ - {false, true}, - {true, false}, - {true, true}, - {false, false} - }; - } - - /** - * This test validates that cursor serializes and deserializes individual-ack list from the bk-ledger. - * @throws Exception - */ - @Test(dataProvider = "unackedRangesOpenCacheSetEnabledPair") - public void testUnackmessagesAndRecoveryCompatibility(boolean enabled1, boolean enabled2) throws Exception { - final String mlName = "ml" + UUID.randomUUID().toString().replaceAll("-", ""); - final String cursorName = "c1"; - ManagedLedgerFactoryConfig factoryConf = new ManagedLedgerFactoryConfig(); - ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc, factoryConf); - final ManagedLedgerConfig config1 = new ManagedLedgerConfig().setEnsembleSize(1).setWriteQuorumSize(1) - .setAckQuorumSize(1).setMetadataEnsembleSize(1).setMetadataWriteQuorumSize(1) - .setMaxUnackedRangesToPersistInMetadataStore(1).setMaxEntriesPerLedger(5).setMetadataAckQuorumSize(1) - .setUnackedRangesOpenCacheSetEnabled(enabled1); - final ManagedLedgerConfig config2 = new ManagedLedgerConfig().setEnsembleSize(1).setWriteQuorumSize(1) - .setAckQuorumSize(1).setMetadataEnsembleSize(1).setMetadataWriteQuorumSize(1) - .setMaxUnackedRangesToPersistInMetadataStore(1).setMaxEntriesPerLedger(5).setMetadataAckQuorumSize(1) - .setUnackedRangesOpenCacheSetEnabled(enabled2); - - ManagedLedger ledger1 = factory.open(mlName, config1); - ManagedCursorImpl cursor1 = (ManagedCursorImpl) ledger1.openCursor(cursorName); - - int totalEntries = 100; - for (int i = 0; i < totalEntries; i++) { - Position p = ledger1.addEntry("entry".getBytes()); - if (i % 2 == 0) { - cursor1.delete(p); - } - } - log.info("ack ranges: {}", cursor1.getIndividuallyDeletedMessagesSet().size()); - - // reopen and recover cursor - ledger1.close(); - ManagedLedger ledger2 = factory.open(mlName, config2); - ManagedCursorImpl cursor2 = (ManagedCursorImpl) ledger2.openCursor(cursorName); - - log.info("before: {}", cursor1.getIndividuallyDeletedMessagesSet().asRanges()); - log.info("after : {}", cursor2.getIndividuallyDeletedMessagesSet().asRanges()); - assertEquals(cursor1.getIndividuallyDeletedMessagesSet().asRanges(), - cursor2.getIndividuallyDeletedMessagesSet().asRanges()); - assertEquals(cursor1.markDeletePosition, cursor2.markDeletePosition); - - ledger2.close(); - factory.shutdown(); - } - @DataProvider(name = "booleans") public Object[][] booleans() { return new Object[][] { @@ -778,7 +722,7 @@ public void testConfigPersistIndividualAckAsLongArray(boolean enable) throws Exc .setEnsembleSize(1).setWriteQuorumSize(1).setAckQuorumSize(1) .setMetadataEnsembleSize(1).setMetadataWriteQuorumSize(1).setMetadataAckQuorumSize(1) .setMaxUnackedRangesToPersistInMetadataStore(1) - .setUnackedRangesOpenCacheSetEnabled(true).setPersistIndividualAckAsLongArray(enable); + .setPersistIndividualAckAsLongArray(enable); ManagedLedger ledger1 = factory.open(mlName, config); ManagedCursorImpl cursor1 = (ManagedCursorImpl) ledger1.openCursor(cursorName); diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetCompatibilityTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetCompatibilityTest.java new file mode 100644 index 0000000000000..474ddd41e5c21 --- /dev/null +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetCompatibilityTest.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.impl; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.function.IntFunction; +import lombok.Cleanup; +import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.ManagedLedgerConfig; +import org.apache.bookkeeper.mledger.ManagedLedgerFactory; +import org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.bookkeeper.mledger.proto.MLDataFormats; +import org.apache.bookkeeper.mledger.proto.MLDataFormats.MessageRange; +import org.apache.bookkeeper.test.BookKeeperClusterTestCase; +import org.apache.pulsar.common.util.collections.LongPairRangeSet; +import org.apache.pulsar.common.util.collections.OpenLongPairRangeSet; +import org.roaringbitmap.RoaringBitSet; +import org.testng.annotations.Test; + +/** + * Comprehensive compatibility tests for PositionRangeSet. + * Covers 4.x to 5.0 upgrade, 5.0 to 4.x downgrade, and cross-ledger MessageRange recovery. + */ +public class PositionRangeSetCompatibilityTest extends BookKeeperClusterTestCase { + + public PositionRangeSetCompatibilityTest() { + super(2); + } + + /** + * Verifies 4.x to 5.0 upgrade with bitmap format. + * Tests that 5.0 PositionRangeSet can deserialize data written by 4.x OpenLongPairRangeSet + * (when unackedRangesOpenCacheSetEnabled=true in 4.x). + */ + @Test + public void testUpgrade_BitmapFormat() { + OpenLongPairRangeSet legacy = new OpenLongPairRangeSet<>(PositionFactory::create); + legacy.addOpenClosed(0, -1, 0, 99); + legacy.addOpenClosed(1, 49, 1, 149); + legacy.addOpenClosed(5, -1, 5, 999); + + Map serialized = legacy.toRanges(Integer.MAX_VALUE); + + PositionRangeSet recovered = new PositionRangeSet(PositionFactory::create, false); + recovered.build(serialized); + + assertEquals(recovered.asRanges(), legacy.asRanges()); + assertEquals(recovered.size(), legacy.size()); + } + + /** + * Verifies 4.x to 5.0 upgrade with MessageRange format including cross-ledger expansion. + * Tests that 5.0 PositionRangeSet can recover data written by 4.x DefaultRangeSet + * (when unackedRangesOpenCacheSetEnabled=false in 4.x) and correctly expands + * MessageRanges that span multiple ledgers using ledger metadata. + */ + @Test + public void testUpgrade_CrossLedgerMessageRange() throws Exception { + String mlName = "test-ml-" + UUID.randomUUID(); + String cursorName = "test-cursor"; + + ManagedLedgerFactoryConfig factoryConf = new ManagedLedgerFactoryConfig(); + @Cleanup("shutdown") + ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc, factoryConf); + + ManagedLedgerConfig config = new ManagedLedgerConfig() + .setEnsembleSize(1).setWriteQuorumSize(1).setAckQuorumSize(1) + .setMetadataEnsembleSize(1).setMetadataWriteQuorumSize(1).setMetadataAckQuorumSize(1) + .setMaxEntriesPerLedger(5); + + ManagedLedger ledger = factory.open(mlName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor(cursorName); + + for (int i = 0; i < 100; i++) { + ledger.addEntry(("entry-" + i).getBytes()); + } + + List messageRanges = new ArrayList<>(); + messageRanges.add(createMessageRange(0, 2, 2, 1)); + messageRanges.add(createMessageRange(5, 0, 7, 3)); + + cursor.recoverIndividualDeletedMessages(messageRanges.size(), (IntFunction) messageRanges::get); + + PositionRangeSet recovered = (PositionRangeSet) cursor.getIndividuallyDeletedMessagesSet(); + + assertTrue(recovered.contains(0, 3)); + assertTrue(recovered.contains(1, 2)); + assertTrue(recovered.contains(2, 1)); + + ledger.close(); + } + + /** + * Verifies 5.0 to 4.x downgrade with bitmap format. + * Tests that 4.x OpenLongPairRangeSet can deserialize data written by 5.0 PositionRangeSet + * (when persistIndividualAckAsLongArray=true in 5.0). + */ + @Test + public void testDowngrade_BitmapFormat() { + PositionRangeSet source = new PositionRangeSet(PositionFactory::create, false); + source.addOpenClosed(0, -1, 0, 99); + source.addOpenClosed(1, 49, 1, 149); + + Map serialized = source.toRanges(Integer.MAX_VALUE); + + OpenLongPairRangeSet legacy = new OpenLongPairRangeSet<>(PositionFactory::create); + legacy.build(serialized); + + assertEquals(legacy.asRanges(), source.asRanges()); + assertEquals(legacy.size(), source.size()); + } + + /** + * Verifies 5.0 to 4.x downgrade with MessageRange format. + * Tests that 4.x DefaultRangeSet can recover data written by 5.0 PositionRangeSet + * (when persistIndividualAckAsLongArray=false in 5.0, which is the default). + */ + @Test + public void testDowngrade_MessageRangeFormat() { + PositionRangeSet source = new PositionRangeSet(PositionFactory::create, false); + source.addOpenClosed(0, -1, 0, 99); + source.addOpenClosed(5, -1, 5, 999); + + List messageRanges = buildMessageRanges(source); + + LongPairRangeSet legacy = + new LongPairRangeSet.DefaultRangeSet<>( + PositionFactory::create, + pos -> new LongPairRangeSet.LongPair(pos.getLedgerId(), pos.getEntryId())); + for (MessageRange mr : messageRanges) { + legacy.addOpenClosed( + mr.getLowerEndpoint().getLedgerId(), mr.getLowerEndpoint().getEntryId(), + mr.getUpperEndpoint().getLedgerId(), mr.getUpperEndpoint().getEntryId()); + } + + assertEquals(legacy.asRanges(), source.asRanges()); + } + + /** + * Verifies bitmap wire format compatibility at binary level. + * Ensures BitSet.toLongArray()/valueOf() produces identical representations + * between 4.x OpenLongPairRangeSet and 5.0 PositionRangeSet. + */ + @Test + public void testBitmapBinaryFormat() { + OpenLongPairRangeSet legacy = new OpenLongPairRangeSet<>( + PositionFactory::create, RoaringBitSet::new); + legacy.addOpenClosed(0, -1, 0, 9); + + Map serialized = legacy.toRanges(Integer.MAX_VALUE); + long[] bitSetArray = serialized.get(0L); + + assertEquals(Arrays.stream(bitSetArray).filter(l -> l != 0).count(), 1); + } + + private MessageRange createMessageRange(long lowerLedger, long lowerEntry, + long upperLedger, long upperEntry) { + return MessageRange.newBuilder() + .setLowerEndpoint(MLDataFormats.NestedPositionInfo.newBuilder() + .setLedgerId(lowerLedger).setEntryId(lowerEntry)) + .setUpperEndpoint(MLDataFormats.NestedPositionInfo.newBuilder() + .setLedgerId(upperLedger).setEntryId(upperEntry)) + .build(); + } + + private List buildMessageRanges(PositionRangeSet rangeSet) { + List result = new ArrayList<>(); + rangeSet.forEachRawRange((lowerKey, lowerValue, upperKey, upperValue) -> { + result.add(createMessageRange(lowerKey, lowerValue, upperKey, upperValue)); + return true; + }); + return result; + } +} diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetTest.java new file mode 100644 index 0000000000000..cccb49678e682 --- /dev/null +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetTest.java @@ -0,0 +1,580 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.impl; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import com.google.common.collect.BoundType; +import com.google.common.collect.Range; +import com.google.common.collect.RangeSet; +import com.google.common.collect.TreeRangeSet; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.commons.lang3.mutable.MutableInt; +import org.apache.pulsar.common.util.collections.LongPairRangeSet.LongPairConsumer; +import org.apache.pulsar.common.util.collections.OpenLongPairRangeSet; +import org.roaringbitmap.RoaringBitSet; +import org.testng.annotations.Test; + +/** + * Tests for {@link PositionRangeSet}, the Position-specific bitmap-backed range set that replaces the + * previous generic {@code RangeSetWrapper} + {@code OpenLongPairRangeSet} stack. + * + *

Originally {@code RangeSetWrapperTest} (parameterized over {@code unackedRangesOpenCacheSetEnabled}). + * The {@code DefaultRangeSet} code path that the {@code false} axis exercised no longer exists, so the + * {@code testAddForDifferentKey2} test and the {@code false} half of {@code testAddForSameKey} / + * {@code testDeleteWithAtMost2} were removed; the remaining assertions reproduce the original + * bitmap-mode ({@code openCacheSet=true}) behavior verbatim. + */ +public class PositionRangeSetTest { + + static final LongPairConsumer CONSUMER = PositionFactory::create; + + private static Position pos(long ledgerId, long entryId) { + return PositionFactory.create(ledgerId, entryId); + } + + // Standard fixture: multi-entry (dirty tracking) enabled — matches ManagedCursorImpl's production wiring. + private static PositionRangeSet newSet() { + return new PositionRangeSet(CONSUMER, true); + } + + @Test + public void testDirtyLedger() { + PositionRangeSet rangeSet = newSet(); + rangeSet.addOpenClosed(10, 0, 20, 0); + assertEquals(rangeSet.size(), 1); + // addOpenClosed(10,0,20,0) marks ledgers in (10, 20] dirty per the original + // dirtyLedgers.addOpenClosed(k,0,k',0) LongPair-ordering semantics — ledger 10 is open-lower + // and therefore not dirty. + assertFalse(rangeSet.isDirtyLedgers(10L)); + for (long i = 11; i <= 20; i++) { + assertTrue(rangeSet.isDirtyLedgers(i)); + } + + rangeSet.removeAtMost(11, 0); + assertEquals(rangeSet.size(), 1); + assertFalse(rangeSet.isDirtyLedgers(11L)); + for (long i = 12; i <= 20; i++) { + assertTrue(rangeSet.isDirtyLedgers(i)); + } + } + + @Test + public void testDirtyLedgerDisabledWhenMultiEntryOff() { + PositionRangeSet rangeSet = new PositionRangeSet(CONSUMER, false); + rangeSet.addOpenClosed(10, 0, 20, 0); + for (long i = 0; i <= 20; i++) { + assertFalse(rangeSet.isDirtyLedgers(i)); + } + } + + @Test + public void testAddForSameKey() { + PositionRangeSet set = newSet(); + set.addOpenClosed(0, 0, 0, 5); + set.addOpenClosed(0, 8, 0, 8); + set.addOpenClosed(0, 9, 0, 9); + set.addOpenClosed(0, 10, 0, 10); + set.addOpenClosed(0, 98, 0, 99); + set.addOpenClosed(0, 102, 0, 106); + + List> ranges = new ArrayList<>(set.asRanges()); + int count = 0; + assertEquals(ranges.get(count++), Range.openClosed(pos(0, 0), pos(0, 5))); + assertEquals(ranges.get(count++), Range.openClosed(pos(0, 98), pos(0, 99))); + assertEquals(ranges.get(count), Range.openClosed(pos(0, 102), pos(0, 106))); + } + + @Test + public void testAddForDifferentKey() { + PositionRangeSet set = newSet(); + set.addOpenClosed(0, 98, 0, 99); + set.addOpenClosed(0, 100, 1, 5); + set.addOpenClosed(1, 10, 1, 15); + set.addOpenClosed(1, 20, 2, 10); + + // bitmap-mode normalization: cross-ledger addOpenClosed into a ledger that did not previously + // exist reports the lower endpoint as (upper, -1) because the bitmap starts at index 0 + // (= entry 0) with the open lower bound at -1. + List> ranges = new ArrayList<>(set.asRanges()); + int count = 0; + assertEquals(ranges.get(count++), Range.openClosed(pos(0, 98), pos(0, 99))); + assertEquals(ranges.get(count++), Range.openClosed(pos(1, -1), pos(1, 5))); + assertEquals(ranges.get(count++), Range.openClosed(pos(1, 10), pos(1, 15))); + assertEquals(ranges.get(count), Range.openClosed(pos(2, -1), pos(2, 10))); + } + + @Test + public void testAddCompareCompareWithGuava() { + PositionRangeSet set = newSet(); + RangeSet gSet = TreeRangeSet.create(); + + int totalInsert = 10_000; + for (int i = 0; i < totalInsert; i++) { + if (i % 3 == 0 || i % 6 == 0 || i % 8 == 0) { + Position lower = pos(0, i - 1); + Position upper = pos(0, i); + set.addOpenClosed(lower.getLedgerId(), lower.getEntryId(), upper.getLedgerId(), upper.getEntryId()); + gSet.add(Range.openClosed(lower, upper)); + } + } + for (int i = totalInsert; i < (totalInsert * 2); i++) { + if (i % 5 == 0) { + Position lower = pos(0, i - 3 - 1); + Position upper = pos(0, i + 3); + set.addOpenClosed(lower.getLedgerId(), lower.getEntryId(), upper.getLedgerId(), upper.getEntryId()); + gSet.add(Range.openClosed(lower, upper)); + } + } + List> ranges = new ArrayList<>(set.asRanges()); + Set> gRanges = gSet.asRanges(); + + List> gRangeConnected = getConnectedRange(gRanges); + assertEquals(gRangeConnected.size(), ranges.size()); + int i = 0; + for (Range range : gRangeConnected) { + assertEquals(range, ranges.get(i)); + i++; + } + } + + @Test + public void testDeleteCompareWithGuava() throws Exception { + PositionRangeSet set = newSet(); + RangeSet gSet = TreeRangeSet.create(); + + int totalInsert = 10_000; + List> removedRanges = new ArrayList<>(); + for (int i = 0; i < totalInsert; i++) { + if (i % 3 == 0 || i % 7 == 0 || i % 11 == 0) { + continue; + } + Position lower = pos(0, i - 1); + Position upper = pos(0, i); + Range range = Range.openClosed(lower, upper); + set.addOpenClosed(lower.getLedgerId(), lower.getEntryId(), upper.getLedgerId(), upper.getEntryId()); + gSet.add(range); + if (i % 4 == 0) { + removedRanges.add(range); + } + } + for (int i = totalInsert; i < (totalInsert * 2); i++) { + Position lower = pos(0, i - 3 - 1); + Position upper = pos(0, i + 3); + Range range = Range.openClosed(lower, upper); + if (i % 5 != 0) { + set.addOpenClosed(lower.getLedgerId(), lower.getEntryId(), upper.getLedgerId(), upper.getEntryId()); + gSet.add(range); + } + if (i % 4 == 0) { + removedRanges.add(range); + } + } + // remove records + for (Range range : removedRanges) { + set.remove(range); + gSet.remove(range); + } + + List> ranges = new ArrayList<>(set.asRanges()); + Set> gRanges = gSet.asRanges(); + List> gRangeConnected = getConnectedRange(gRanges); + assertEquals(gRangeConnected.size(), ranges.size()); + int i = 0; + for (Range range : gRangeConnected) { + assertEquals(range, ranges.get(i)); + i++; + } + } + + @Test + public void testSpanWithGuava() { + PositionRangeSet set = newSet(); + RangeSet gSet = TreeRangeSet.create(); + set.addOpenClosed(0, 97, 0, 99); + gSet.add(Range.openClosed(pos(0, 97), pos(0, 99))); + set.addOpenClosed(0, 99, 1, 5); + gSet.add(Range.openClosed(pos(0, 99), pos(1, 5))); + assertEquals(set.span(), gSet.span()); + assertEquals(set.span(), Range.openClosed(pos(0, 97), pos(1, 5))); + + set.addOpenClosed(1, 9, 1, 15); + set.addOpenClosed(1, 19, 2, 10); + set.addOpenClosed(2, 24, 2, 28); + set.addOpenClosed(3, 11, 3, 20); + set.addOpenClosed(4, 11, 4, 20); + gSet.add(Range.openClosed(pos(1, 9), pos(1, 15))); + gSet.add(Range.openClosed(pos(1, 19), pos(2, 10))); + gSet.add(Range.openClosed(pos(2, 24), pos(2, 28))); + gSet.add(Range.openClosed(pos(3, 11), pos(3, 20))); + gSet.add(Range.openClosed(pos(4, 11), pos(4, 20))); + assertEquals(set.span(), gSet.span()); + assertEquals(set.span(), Range.openClosed(pos(0, 97), pos(4, 20))); + } + + @Test + public void testFirstRange() { + PositionRangeSet set = newSet(); + assertNull(set.firstRange()); + set.addOpenClosed(0, 97, 0, 99); + assertEquals(set.firstRange(), Range.openClosed(pos(0, 97), pos(0, 99))); + assertEquals(set.size(), 1); + set.addOpenClosed(0, 98, 0, 105); + assertEquals(set.firstRange(), Range.openClosed(pos(0, 97), pos(0, 105))); + assertEquals(set.size(), 1); + set.addOpenClosed(0, 5, 0, 75); + assertEquals(set.firstRange(), Range.openClosed(pos(0, 5), pos(0, 75))); + assertEquals(set.size(), 2); + } + + @Test + public void testLastRange() { + PositionRangeSet set = newSet(); + assertNull(set.lastRange()); + Range range = Range.openClosed(pos(0, 97), pos(0, 99)); + set.addOpenClosed(0, 97, 0, 99); + assertEquals(set.lastRange(), range); + assertEquals(set.size(), 1); + set.addOpenClosed(0, 98, 0, 105); + assertEquals(set.lastRange(), Range.openClosed(pos(0, 97), pos(0, 105))); + assertEquals(set.size(), 1); + range = Range.openClosed(pos(1, 5), pos(1, 75)); + set.addOpenClosed(1, 5, 1, 75); + assertEquals(set.lastRange(), range); + assertEquals(set.size(), 2); + range = Range.openClosed(pos(1, 80), pos(1, 120)); + set.addOpenClosed(1, 80, 1, 120); + assertEquals(set.lastRange(), range); + assertEquals(set.size(), 3); + } + + @Test + public void testToString() { + PositionRangeSet set = newSet(); + set.addOpenClosed(0, 97, 0, 99); + assertEquals(set.toString(), "[(0:97..0:99]]"); + set.addOpenClosed(0, 98, 0, 105); + assertEquals(set.toString(), "[(0:97..0:105]]"); + set.addOpenClosed(0, 5, 0, 75); + assertEquals(set.toString(), "[(0:5..0:75],(0:97..0:105]]"); + } + + @Test + public void testDeleteForDifferentKey() { + PositionRangeSet set = newSet(); + set.addOpenClosed(0, 97, 0, 99); + set.addOpenClosed(0, 99, 1, 5); + set.addOpenClosed(1, 9, 1, 15); + set.addOpenClosed(1, 19, 2, 10); + set.addOpenClosed(2, 24, 2, 28); + set.addOpenClosed(3, 11, 3, 20); + set.addOpenClosed(4, 11, 4, 20); + + // delete only (0,100) + set.remove(Range.open(pos(0, 99), pos(0, 105))); + + /** + * delete all keys from [2,27]->[4,15] : remaining [2,25..26,28], [4,16..20] + */ + set.remove(Range.closed(pos(2, 27), pos(4, 15))); + + List> ranges = new ArrayList<>(set.asRanges()); + int count = 0; + assertEquals(ranges.get(count++), Range.openClosed(pos(0, 97), pos(0, 99))); + assertEquals(ranges.get(count++), Range.openClosed(pos(1, -1), pos(1, 5))); + assertEquals(ranges.get(count++), Range.openClosed(pos(1, 9), pos(1, 15))); + assertEquals(ranges.get(count++), Range.openClosed(pos(2, -1), pos(2, 10))); + + assertEquals(ranges.get(count++), Range.openClosed(pos(2, 24), pos(2, 26))); + assertEquals(ranges.get(count++), Range.openClosed(pos(2, 27), pos(2, 28))); + assertEquals(ranges.get(count++), Range.openClosed(pos(4, 15), pos(4, 20))); + } + + @Test + public void testDeleteWithAtMost() { + PositionRangeSet set = newSet(); + set.addOpenClosed(0, 98, 0, 99); + set.addOpenClosed(0, 100, 1, 5); + set.addOpenClosed(1, 10, 1, 15); + set.addOpenClosed(1, 20, 2, 10); + set.addOpenClosed(2, 25, 2, 28); + set.addOpenClosed(3, 12, 3, 20); + set.addOpenClosed(4, 12, 4, 20); + + // delete only (0,100) + set.remove(Range.open(pos(0, 99), pos(0, 105))); + + /** + * delete all keys from [2,27]->[4,15] : remaining [2,25..26,28], [4,16..20] + */ + set.remove(Range.atMost(pos(2, 27))); + + List> ranges = new ArrayList<>(set.asRanges()); + int count = 0; + assertEquals(ranges.get(count++), Range.openClosed(pos(2, 27), pos(2, 28))); + assertEquals(ranges.get(count++), Range.openClosed(pos(3, 12), pos(3, 20))); + assertEquals(ranges.get(count++), Range.openClosed(pos(4, 12), pos(4, 20))); + } + + @Test + public void testDeleteWithAtMost2() { + // Originally this test ran twice — once with openCacheSet=true and once with =false. + // The =false (DefaultRangeSet) variant is dropped because PositionRangeSet has a single + // bitmap-backed implementation. The remaining assertions reproduce the bitmap-mode behavior. + PositionRangeSet set = newSet(); + set.addOpenClosed(0, 98, 0, 99); + set.addOpenClosed(0, 100, 1, 5); + set.addOpenClosed(1, 10, 1, 15); + set.addOpenClosed(1, 20, 2, 10); + set.addOpenClosed(2, 25, 2, 28); + set.addOpenClosed(3, 12, 3, 20); + set.addOpenClosed(4, 12, 4, 20); + + // delete entire ledger 0 (closed-closed within the same ledger id) + set.remove(Range.closed(pos(0, 0), pos(0, Integer.MAX_VALUE - 1))); + + List> ranges = new ArrayList<>(set.asRanges()); + int count = 0; + assertEquals(ranges.get(count++), Range.openClosed(pos(1, -1), pos(1, 5))); + assertEquals(ranges.get(count++), Range.openClosed(pos(1, 10), pos(1, 15))); + assertEquals(ranges.get(count++), Range.openClosed(pos(2, -1), pos(2, 10))); + assertEquals(ranges.get(count), Range.openClosed(pos(2, 25), pos(2, 28))); + } + + @Test + public void testDeleteWithLeastMost() { + PositionRangeSet set = newSet(); + set.addOpenClosed(0, 98, 0, 99); + set.addOpenClosed(0, 100, 1, 5); + set.addOpenClosed(1, 10, 1, 15); + set.addOpenClosed(1, 20, 2, 10); + set.addOpenClosed(2, 25, 2, 28); + set.addOpenClosed(2, 12, 3, 20); + set.addOpenClosed(4, 12, 4, 20); + + // delete only (0,100) + set.remove(Range.open(pos(0, 99), pos(0, 105))); + + /** + * delete all keys from [2,27]->[4,15] : remaining [2,25..26,28], [4,16..20] + */ + set.remove(Range.atLeast(pos(2, 27))); + + List> ranges = new ArrayList<>(set.asRanges()); + int count = 0; + assertEquals(ranges.get(count++), Range.openClosed(pos(0, 98), pos(0, 99))); + assertEquals(ranges.get(count++), Range.openClosed(pos(1, -1), pos(1, 5))); + assertEquals(ranges.get(count++), Range.openClosed(pos(1, 10), pos(1, 15))); + assertEquals(ranges.get(count++), Range.openClosed(pos(2, -1), pos(2, 10))); + assertEquals(ranges.get(count), Range.openClosed(pos(2, 12), pos(2, 26))); + } + + @Test + public void testRangeContaining() { + PositionRangeSet set = newSet(); + set.add(Range.closed(pos(0, 98), pos(0, 99))); + set.add(Range.closed(pos(0, 100), pos(1, 5))); + RangeSet gSet = TreeRangeSet.create(); + gSet.add(Range.closed(pos(0, 98), pos(0, 100))); + gSet.add(Range.closed(pos(0, 101), pos(1, 5))); + set.add(Range.closed(pos(1, 10), pos(1, 15))); + set.add(Range.closed(pos(1, 20), pos(2, 10))); + set.add(Range.closed(pos(2, 25), pos(2, 28))); + set.add(Range.closed(pos(3, 12), pos(3, 20))); + set.add(Range.closed(pos(4, 12), pos(4, 20))); + gSet.add(Range.closed(pos(1, 10), pos(1, 15))); + gSet.add(Range.closed(pos(1, 20), pos(2, 10))); + gSet.add(Range.closed(pos(2, 25), pos(2, 28))); + gSet.add(Range.closed(pos(3, 12), pos(3, 20))); + gSet.add(Range.closed(pos(4, 12), pos(4, 20))); + + Position position = pos(0, 99); + assertEquals(set.rangeContaining(position.getLedgerId(), position.getEntryId()), + Range.closed(pos(0, 98), pos(0, 100))); + assertEquals(set.rangeContaining(position.getLedgerId(), position.getEntryId()), + gSet.rangeContaining(position)); + + position = pos(2, 30); + assertNull(set.rangeContaining(position.getLedgerId(), position.getEntryId())); + assertEquals(set.rangeContaining(position.getLedgerId(), position.getEntryId()), + gSet.rangeContaining(position)); + + position = pos(3, 13); + assertEquals(set.rangeContaining(position.getLedgerId(), position.getEntryId()), + Range.closed(pos(3, 12), pos(3, 20))); + assertEquals(set.rangeContaining(position.getLedgerId(), position.getEntryId()), + gSet.rangeContaining(position)); + + position = pos(3, 22); + assertNull(set.rangeContaining(position.getLedgerId(), position.getEntryId())); + assertEquals(set.rangeContaining(position.getLedgerId(), position.getEntryId()), + gSet.rangeContaining(position)); + } + + @Test + public void testWireFormatRoundTrip() { + // Verify the dense-long[] contract used by ManagedCursorImpl persistence: serialize via + // toRanges, deserialize via build, and confirm the resulting range set is identical. + PositionRangeSet original = newSet(); + original.addOpenClosed(0, 0, 0, 5); + original.addOpenClosed(0, 10, 0, 100); + original.addOpenClosed(1, 0, 1, 50); + original.addOpenClosed(3, 200, 3, 300); + + Map serialized = original.toRanges(Integer.MAX_VALUE); + PositionRangeSet restored = newSet(); + restored.build(serialized); + + assertEquals(restored.asRanges(), original.asRanges()); + assertEquals(restored.cardinality(0, 0, 3, 300), original.cardinality(0, 0, 3, 300)); + for (Range r : original.asRanges()) { + assertTrue(restored.contains(r.lowerEndpoint().getLedgerId(), r.lowerEndpoint().getEntryId() + 1)); + } + } + + @Test + public void testCompatibilityWithLegacyOpenLongPairRangeSet() { + // Simulate 4.x cursor persistence: serialize individual deleted messages using the legacy + // OpenLongPairRangeSet (still present in pulsar-common) backed by RoaringBitSet, then + // deserialize via PositionRangeSet. This proves on-disk byte compatibility — a cursor + // persisted by 4.x can be recovered by 5.0 without conversion. + OpenLongPairRangeSet legacy = new OpenLongPairRangeSet<>( + PositionFactory::create, RoaringBitSet::new); + legacy.addOpenClosed(0, -1, 0, 9); + legacy.addOpenClosed(0, 50, 0, 99); + legacy.addOpenClosed(1, 10, 1, 20); + legacy.addOpenClosed(3, 0, 4, 5); + + Map legacySerialized = legacy.toRanges(Integer.MAX_VALUE); + PositionRangeSet restored = newSet(); + restored.build(legacySerialized); + + assertEquals(restored.asRanges(), legacy.asRanges()); + assertEquals(restored.size(), legacy.size()); + assertEquals(restored.cardinality(0, 0, 4, 5), legacy.cardinality(0, 0, 4, 5)); + } + + @Test + public void testCardinality() { + PositionRangeSet set = newSet(); + // half-open lower: addOpenClosed(k, -1, k, N) sets entries 0..N inclusive. + set.addOpenClosed(0, -1, 0, 9); // 10 entries: 0..9 + set.addOpenClosed(1, -1, 1, 19); // 20 entries: 0..19 + set.addOpenClosed(2, -1, 2, 4); // 5 entries: 0..4 + + // full span — cardinality bounds are inclusive on both ends + assertEquals(set.cardinality(0, 0, 2, 4), 35); + // partial ledger 0 + assertEquals(set.cardinality(0, 3, 0, 7), 5); + // partial across ledgers 0 and 1 + assertEquals(set.cardinality(0, 5, 1, 5), 5 + 6); + // single ledger interior + assertEquals(set.cardinality(1, 5, 1, 14), 10); + } + + @Test + public void testClear() { + PositionRangeSet set = newSet(); + // cross-ledger adds mark dirty (single-ledger adds do not, matching the original + // RangeSetWrapper/DefaultRangeSet semantics where addOpenClosed(k,0,k,0) is an empty range). + set.addOpenClosed(0, 5, 1, 5); + set.addOpenClosed(2, 5, 3, 5); + assertFalse(set.isEmpty()); + assertEquals(set.size(), 2); + assertTrue(set.isDirtyLedgers(1)); + assertTrue(set.isDirtyLedgers(3)); + set.clear(); + assertTrue(set.isEmpty()); + assertEquals(set.size(), 0); + // clear also resets dirty tracker + assertFalse(set.isDirtyLedgers(1)); + assertFalse(set.isDirtyLedgers(3)); + } + + @Test + public void testRemoveAtMostClearsEmptyLedgers() { + // Cross-ledger removeAtMost must drop now-empty ledgers from the underlying TreeMap so + // the structure does not accumulate stale empty bitmaps over the cursor's lifetime. + PositionRangeSet set = newSet(); + set.addOpenClosed(0, -1, 0, 9); // ledger 0: entries 0..9 + set.addOpenClosed(1, -1, 1, 9); // ledger 1: entries 0..9 + set.addOpenClosed(2, -1, 2, 9); // ledger 2: entries 0..9 + + // removeAtMost(2, -1) deletes ledgers 0 and 1 wholesale (positions <= (2, -1)) and leaves + // ledger 2 untouched (no entry <= -1 in ledger 2). + set.removeAtMost(2, -1); + + // Only ledger 2 should remain in the underlying map. + List> ranges = new ArrayList<>(set.asRanges()); + assertEquals(ranges.size(), 1); + assertEquals(ranges.get(0), Range.openClosed(pos(2, -1), pos(2, 9))); + // The two cleared ledgers must not linger as empty bitmaps — verify via forEachRawRange, + // which would otherwise skip them silently and hide the leak. + MutableInt ledgerCount = new MutableInt(0); + set.forEachRawRange((lowerKey, lowerValue, upperKey, upperValue) -> { + ledgerCount.increment(); + return true; + }); + assertEquals(ledgerCount.intValue(), 1); + } + + + private List> getConnectedRange(Set> gRanges) { + List> gRangeConnected = new ArrayList<>(); + Range lastRange = null; + for (Range range : gRanges) { + if (lastRange == null) { + lastRange = range; + continue; + } + Position previousUpper = lastRange.upperEndpoint(); + Position currentLower = range.lowerEndpoint(); + int previousUpperValue = (int) (lastRange.upperBoundType().equals(BoundType.CLOSED) + ? previousUpper.getEntryId() + : previousUpper.getEntryId() - 1); + int currentLowerValue = (int) (range.lowerBoundType().equals(BoundType.CLOSED) + ? currentLower.getEntryId() + : currentLower.getEntryId() + 1); + boolean connected = previousUpper.getLedgerId() == currentLower.getLedgerId() + && (previousUpperValue >= currentLowerValue); + if (connected) { + lastRange = Range.closed(lastRange.lowerEndpoint(), range.upperEndpoint()); + } else { + gRangeConnected.add(lastRange); + lastRange = range; + } + } + if (lastRange != null) { + int lowerOpenValue = (int) (lastRange.lowerBoundType().equals(BoundType.CLOSED) + ? (lastRange.lowerEndpoint().getEntryId() - 1) + : lastRange.lowerEndpoint().getEntryId()); + lastRange = Range.openClosed(pos(lastRange.lowerEndpoint().getLedgerId(), lowerOpenValue), + lastRange.upperEndpoint()); + gRangeConnected.add(lastRange); + } + return gRangeConnected; + } +} diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/RangeSetWrapperTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/RangeSetWrapperTest.java deleted file mode 100644 index 9012945ddf7c1..0000000000000 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/RangeSetWrapperTest.java +++ /dev/null @@ -1,516 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.bookkeeper.mledger.impl; - -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNull; -import static org.testng.Assert.assertTrue; -import com.google.common.collect.BoundType; -import com.google.common.collect.Range; -import com.google.common.collect.TreeRangeSet; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import org.apache.bookkeeper.mledger.ManagedLedgerConfig; -import org.apache.pulsar.common.util.collections.LongPairRangeSet.LongPair; -import org.apache.pulsar.common.util.collections.LongPairRangeSet.LongPairConsumer; -import org.apache.pulsar.common.util.collections.LongPairRangeSet.RangeBoundConsumer; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -public class RangeSetWrapperTest { - - static final LongPairConsumer CONSUMER = (key, value) -> new LongPair(key, value); - static final RangeBoundConsumer REVERSE_CONVERT = (pair) -> pair; - - ManagedLedgerImpl managedLedger; - RangeSetWrapper set; - ManagedLedgerConfig managedLedgerConfig; - ManagedCursorImpl managedCursor; - - @BeforeMethod - public void setUp() { - initManagedLedgerConfig(); - managedLedger = mock(ManagedLedgerImpl.class); - managedCursor = mock(ManagedCursorImpl.class); - doReturn(managedLedgerConfig).when(managedLedger).getConfig(); - doReturn(managedLedgerConfig).when(managedCursor).getConfig(); - doReturn(managedLedger).when(managedCursor).getManagedLedger(); - } - - private void initManagedLedgerConfig() { - managedLedgerConfig = new ManagedLedgerConfig(); - managedLedgerConfig.setUnackedRangesOpenCacheSetEnabled(true); - managedLedgerConfig.setPersistentUnackedRangesWithMultipleEntriesEnabled(true); - } - - @AfterMethod - public void clean() throws Exception { - } - - @Test - public void testDirtyLedger() { - RangeSetWrapper rangeSetWrapper = new RangeSetWrapper<>(CONSUMER, - REVERSE_CONVERT, - managedCursor); - // Test add range - rangeSetWrapper.addOpenClosed(10, 0, 20, 0); - assertEquals(rangeSetWrapper.size(), 1); - assertFalse(rangeSetWrapper.isDirtyLedgers(10L)); - for (long i = 11; i < 20; i++) { - assertTrue(rangeSetWrapper.isDirtyLedgers(i)); - } - - // Test remove range - rangeSetWrapper.removeAtMost(11, 0); - assertEquals(rangeSetWrapper.size(), 1); - assertFalse(rangeSetWrapper.isDirtyLedgers(11L)); - for (long i = 12; i < 20; i++) { - assertTrue(rangeSetWrapper.isDirtyLedgers(i)); - } - } - - @Test - public void testAddForSameKey() { - doTestAddForSameKey(); - managedLedgerConfig.setUnackedRangesOpenCacheSetEnabled(false); - doTestAddForSameKey(); - } - - private void doTestAddForSameKey() { - set = new RangeSetWrapper(CONSUMER, REVERSE_CONVERT, managedCursor); - // add 0 to 5 - set.addOpenClosed(0, 0, 0, 5); - // add 8,9,10 - set.addOpenClosed(0, 8, 0, 8); - set.addOpenClosed(0, 9, 0, 9); - set.addOpenClosed(0, 10, 0, 10); - // add 98 to 99 and 102,105 - set.addOpenClosed(0, 98, 0, 99); - set.addOpenClosed(0, 102, 0, 106); - - List> ranges = new ArrayList<>(set.asRanges()); - int count = 0; - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(0, 0), new LongPair(0, 5)))); - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(0, 98), new LongPair(0, 99)))); - assertEquals(ranges.get(count), (Range.openClosed(new LongPair(0, 102), new LongPair(0, 106)))); - } - - @Test - public void testAddForDifferentKey() { - set = new RangeSetWrapper<>(CONSUMER, REVERSE_CONVERT, managedCursor); - // [98,100],[(1,5),(1,5)],[(1,10,1,15)],[(1,20),(1,20)],[(2,0),(2,10)] - set.addOpenClosed(0, 98, 0, 99); - set.addOpenClosed(0, 100, 1, 5); - set.addOpenClosed(1, 10, 1, 15); - set.addOpenClosed(1, 20, 2, 10); - - List> ranges = new ArrayList<>(set.asRanges()); - int count = 0; - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(0, 98), new LongPair(0, 99)))); - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(1, -1), new LongPair(1, 5)))); - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(1, 10), new LongPair(1, 15)))); - assertEquals(ranges.get(count), (Range.openClosed(new LongPair(2, -1), new LongPair(2, 10)))); - } - - @Test - public void testAddForDifferentKey2() { - managedLedgerConfig.setUnackedRangesOpenCacheSetEnabled(false); - set = new RangeSetWrapper<>(CONSUMER, REVERSE_CONVERT, managedCursor); - // [98,100],[(1,5),(1,5)],[(1,10,1,15)],[(1,20),(1,20)],[(2,0),(2,10)] - set.addOpenClosed(0, 98, 0, 99); - set.addOpenClosed(0, 100, 1, 5); - set.addOpenClosed(1, 10, 1, 15); - set.addOpenClosed(1, 20, 2, 10); - - List> ranges = new ArrayList<>(set.asRanges()); - int count = 0; - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(0, 98), new LongPair(0, 99)))); - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(0, 100), new LongPair(1, 5)))); - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(1, 10), new LongPair(1, 15)))); - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(1, 20), new LongPair(2, 10)))); - } - - @Test - public void testAddCompareCompareWithGuava() { - set = new RangeSetWrapper<>(CONSUMER, REVERSE_CONVERT, managedCursor); - com.google.common.collect.RangeSet gSet = TreeRangeSet.create(); - - // add 10K values for key 0 - int totalInsert = 10_000; - // add single values - for (int i = 0; i < totalInsert; i++) { - if (i % 3 == 0 || i % 6 == 0 || i % 8 == 0) { - LongPair lower = new LongPair(0, i - 1); - LongPair upper = new LongPair(0, i); - // set.add(Range.openClosed(lower, upper)); - set.addOpenClosed(lower.getKey(), lower.getValue(), upper.getKey(), upper.getValue()); - gSet.add(Range.openClosed(lower, upper)); - } - } - // add batches - for (int i = totalInsert; i < (totalInsert * 2); i++) { - if (i % 5 == 0) { - LongPair lower = new LongPair(0, i - 3 - 1); - LongPair upper = new LongPair(0, i + 3); - // set.add(Range.openClosed(lower, upper)); - set.addOpenClosed(lower.getKey(), lower.getValue(), upper.getKey(), upper.getValue()); - gSet.add(Range.openClosed(lower, upper)); - } - } - List> ranges = new ArrayList<>(set.asRanges()); - Set> gRanges = gSet.asRanges(); - - List> gRangeConnected = getConnectedRange(gRanges); - assertEquals(gRangeConnected.size(), ranges.size()); - int i = 0; - for (Range range : gRangeConnected) { - assertEquals(range, ranges.get(i)); - i++; - } - } - - @Test - public void testDeleteCompareWithGuava() throws Exception { - RangeSetWrapper set = new RangeSetWrapper<>(CONSUMER, REVERSE_CONVERT, managedCursor); - com.google.common.collect.RangeSet gSet = TreeRangeSet.create(); - - // add 10K values for key 0 - int totalInsert = 10_000; - // add single values - List> removedRanges = new ArrayList(); - for (int i = 0; i < totalInsert; i++) { - if (i % 3 == 0 || i % 7 == 0 || i % 11 == 0) { - continue; - } - LongPair lower = new LongPair(0, i - 1); - LongPair upper = new LongPair(0, i); - Range range = Range.openClosed(lower, upper); - // set.add(range); - set.addOpenClosed(lower.getKey(), lower.getValue(), upper.getKey(), upper.getValue()); - gSet.add(range); - if (i % 4 == 0) { - removedRanges.add(range); - } - } - // add batches - for (int i = totalInsert; i < (totalInsert * 2); i++) { - LongPair lower = new LongPair(0, i - 3 - 1); - LongPair upper = new LongPair(0, i + 3); - Range range = Range.openClosed(lower, upper); - if (i % 5 != 0) { - // set.add(range); - set.addOpenClosed(lower.getKey(), lower.getValue(), upper.getKey(), upper.getValue()); - gSet.add(range); - } - if (i % 4 == 0) { - removedRanges.add(range); - } - } - // remove records - for (Range range : removedRanges) { - set.remove(range); - gSet.remove(range); - } - - List> ranges = new ArrayList<>(set.asRanges()); - Set> gRanges = gSet.asRanges(); - List> gRangeConnected = getConnectedRange(gRanges); - assertEquals(gRangeConnected.size(), ranges.size()); - int i = 0; - for (Range range : gRangeConnected) { - assertEquals(range, ranges.get(i)); - i++; - } - } - - @Test - public void testSpanWithGuava() { - set = new RangeSetWrapper<>(CONSUMER, REVERSE_CONVERT, managedCursor); - com.google.common.collect.RangeSet gSet = TreeRangeSet.create(); - set.addOpenClosed(0, 97, 0, 99); - gSet.add(Range.openClosed(new LongPair(0, 97), new LongPair(0, 99))); - set.addOpenClosed(0, 99, 1, 5); - gSet.add(Range.openClosed(new LongPair(0, 99), new LongPair(1, 5))); - assertEquals(set.span(), gSet.span()); - assertEquals(set.span(), Range.openClosed(new LongPair(0, 97), new LongPair(1, 5))); - - set.addOpenClosed(1, 9, 1, 15); - set.addOpenClosed(1, 19, 2, 10); - set.addOpenClosed(2, 24, 2, 28); - set.addOpenClosed(3, 11, 3, 20); - set.addOpenClosed(4, 11, 4, 20); - gSet.add(Range.openClosed(new LongPair(1, 9), new LongPair(1, 15))); - gSet.add(Range.openClosed(new LongPair(1, 19), new LongPair(2, 10))); - gSet.add(Range.openClosed(new LongPair(2, 24), new LongPair(2, 28))); - gSet.add(Range.openClosed(new LongPair(3, 11), new LongPair(3, 20))); - gSet.add(Range.openClosed(new LongPair(4, 11), new LongPair(4, 20))); - assertEquals(set.span(), gSet.span()); - assertEquals(set.span(), Range.openClosed(new LongPair(0, 97), new LongPair(4, 20))); - } - - @Test - public void testFirstRange() { - set = new RangeSetWrapper<>(CONSUMER, REVERSE_CONVERT, managedCursor); - assertNull(set.firstRange()); - set.addOpenClosed(0, 97, 0, 99); - assertEquals(set.firstRange(), Range.openClosed(new LongPair(0, 97), new LongPair(0, 99))); - assertEquals(set.size(), 1); - set.addOpenClosed(0, 98, 0, 105); - assertEquals(set.firstRange(), Range.openClosed(new LongPair(0, 97), new LongPair(0, 105))); - assertEquals(set.size(), 1); - set.addOpenClosed(0, 5, 0, 75); - assertEquals(set.firstRange(), Range.openClosed(new LongPair(0, 5), new LongPair(0, 75))); - assertEquals(set.size(), 2); - } - - @Test - public void testLastRange() { - set = new RangeSetWrapper<>(CONSUMER, REVERSE_CONVERT, managedCursor); - assertNull(set.lastRange()); - Range range = Range.openClosed(new LongPair(0, 97), new LongPair(0, 99)); - set.addOpenClosed(0, 97, 0, 99); - assertEquals(set.lastRange(), range); - assertEquals(set.size(), 1); - set.addOpenClosed(0, 98, 0, 105); - assertEquals(set.lastRange(), Range.openClosed(new LongPair(0, 97), new LongPair(0, 105))); - assertEquals(set.size(), 1); - range = Range.openClosed(new LongPair(1, 5), new LongPair(1, 75)); - set.addOpenClosed(1, 5, 1, 75); - assertEquals(set.lastRange(), range); - assertEquals(set.size(), 2); - range = Range.openClosed(new LongPair(1, 80), new LongPair(1, 120)); - set.addOpenClosed(1, 80, 1, 120); - assertEquals(set.lastRange(), range); - assertEquals(set.size(), 3); - } - - @Test - public void testToString() { - set = new RangeSetWrapper<>(CONSUMER, REVERSE_CONVERT, managedCursor); - set.addOpenClosed(0, 97, 0, 99); - assertEquals(set.toString(), "[(0:97..0:99]]"); - set.addOpenClosed(0, 98, 0, 105); - assertEquals(set.toString(), "[(0:97..0:105]]"); - set.addOpenClosed(0, 5, 0, 75); - assertEquals(set.toString(), "[(0:5..0:75],(0:97..0:105]]"); - } - - @Test - public void testDeleteForDifferentKey() { - set = new RangeSetWrapper<>(CONSUMER, REVERSE_CONVERT, managedCursor); - set.addOpenClosed(0, 97, 0, 99); - set.addOpenClosed(0, 99, 1, 5); - set.addOpenClosed(1, 9, 1, 15); - set.addOpenClosed(1, 19, 2, 10); - set.addOpenClosed(2, 24, 2, 28); - set.addOpenClosed(3, 11, 3, 20); - set.addOpenClosed(4, 11, 4, 20); - - // delete only (0,100) - set.remove(Range.open(new LongPair(0, 99), new LongPair(0, 105))); - - /** - * delete all keys from [2,27]->[4,15] : remaining [2,25..26,28], [4,16..20] - */ - set.remove(Range.closed(new LongPair(2, 27), new LongPair(4, 15))); - - List> ranges = new ArrayList<>(set.asRanges()); - int count = 0; - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(0, 97), new LongPair(0, 99)))); - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(1, -1), new LongPair(1, 5)))); - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(1, 9), new LongPair(1, 15)))); - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(2, -1), new LongPair(2, 10)))); - - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(2, 24), new LongPair(2, 26)))); - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(2, 27), new LongPair(2, 28)))); - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(4, 15), new LongPair(4, 20)))); - } - - @Test - public void testDeleteWithAtMost() { - set = new RangeSetWrapper<>(CONSUMER, REVERSE_CONVERT, managedCursor); - set.addOpenClosed(0, 98, 0, 99); - set.addOpenClosed(0, 100, 1, 5); - set.addOpenClosed(1, 10, 1, 15); - set.addOpenClosed(1, 20, 2, 10); - set.addOpenClosed(2, 25, 2, 28); - set.addOpenClosed(3, 12, 3, 20); - set.addOpenClosed(4, 12, 4, 20); - - // delete only (0,100) - set.remove(Range.open(new LongPair(0, 99), new LongPair(0, 105))); - - /** - * delete all keys from [2,27]->[4,15] : remaining [2,25..26,28], [4,16..20] - */ - set.remove(Range.atMost(new LongPair(2, 27))); - - List> ranges = new ArrayList<>(set.asRanges()); - int count = 0; - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(2, 27), new LongPair(2, 28)))); - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(3, 12), new LongPair(3, 20)))); - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(4, 12), new LongPair(4, 20)))); - } - - @Test - public void testDeleteWithAtMost2() { - set = new RangeSetWrapper<>(CONSUMER, REVERSE_CONVERT, managedCursor); - set.addOpenClosed(0, 98, 0, 99); - set.addOpenClosed(0, 100, 1, 5); - set.addOpenClosed(1, 10, 1, 15); - set.addOpenClosed(1, 20, 2, 10); - set.addOpenClosed(2, 25, 2, 28); - set.addOpenClosed(3, 12, 3, 20); - set.addOpenClosed(4, 12, 4, 20); - - // delete only (0,100) - set.remove(Range.closed(new LongPair(0, 0), new LongPair(0, Integer.MAX_VALUE - 1))); - - List> ranges = new ArrayList<>(set.asRanges()); - int count = 0; - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(1, -1), new LongPair(1, 5)))); - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(1, 10), new LongPair(1, 15)))); - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(2, -1), new LongPair(2, 10)))); - assertEquals(ranges.get(count), (Range.openClosed(new LongPair(2, 25), new LongPair(2, 28)))); - - managedLedgerConfig.setUnackedRangesOpenCacheSetEnabled(false); - set = new RangeSetWrapper<>(CONSUMER, REVERSE_CONVERT, managedCursor); - set.addOpenClosed(0, 98, 0, 99); - set.addOpenClosed(0, 100, 1, 5); - set.addOpenClosed(1, 10, 1, 15); - set.addOpenClosed(1, 20, 2, 10); - set.addOpenClosed(2, 25, 2, 28); - set.addOpenClosed(3, 12, 3, 20); - set.addOpenClosed(4, 12, 4, 20); - - set.remove(Range.openClosed(new LongPair(0, 0), new LongPair(0, Integer.MAX_VALUE - 1))); - ranges = new ArrayList<>(set.asRanges()); - count = 0; - assertEquals(ranges.get(count++), - (Range.openClosed(new LongPair(0, Integer.MAX_VALUE - 1), new LongPair(1, 5)))); - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(1, 10), new LongPair(1, 15)))); - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(1, 20), new LongPair(2, 10)))); - assertEquals(ranges.get(count), (Range.openClosed(new LongPair(2, 25), new LongPair(2, 28)))); - } - - @Test - public void testDeleteWithLeastMost() { - set = new RangeSetWrapper<>(CONSUMER, REVERSE_CONVERT, managedCursor); - set.addOpenClosed(0, 98, 0, 99); - set.addOpenClosed(0, 100, 1, 5); - set.addOpenClosed(1, 10, 1, 15); - set.addOpenClosed(1, 20, 2, 10); - set.addOpenClosed(2, 25, 2, 28); - set.addOpenClosed(2, 12, 3, 20); - set.addOpenClosed(4, 12, 4, 20); - - // delete only (0,100) - set.remove(Range.open(new LongPair(0, 99), new LongPair(0, 105))); - - /** - * delete all keys from [2,27]->[4,15] : remaining [2,25..26,28], [4,16..20] - */ - set.remove(Range.atLeast(new LongPair(2, 27))); - - List> ranges = new ArrayList<>(set.asRanges()); - int count = 0; - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(0, 98), new LongPair(0, 99)))); - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(1, -1), new LongPair(1, 5)))); - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(1, 10), new LongPair(1, 15)))); - assertEquals(ranges.get(count++), (Range.openClosed(new LongPair(2, -1), new LongPair(2, 10)))); - assertEquals(ranges.get(count), (Range.openClosed(new LongPair(2, 12), new LongPair(2, 26)))); - } - - @Test - public void testRangeContaining() { - set = new RangeSetWrapper<>(CONSUMER, REVERSE_CONVERT, managedCursor); - set.add(Range.closed(new LongPair(0, 98), new LongPair(0, 99))); - set.add(Range.closed(new LongPair(0, 100), new LongPair(1, 5))); - com.google.common.collect.RangeSet gSet = TreeRangeSet.create(); - gSet.add(Range.closed(new LongPair(0, 98), new LongPair(0, 100))); - gSet.add(Range.closed(new LongPair(0, 101), new LongPair(1, 5))); - set.add(Range.closed(new LongPair(1, 10), new LongPair(1, 15))); - set.add(Range.closed(new LongPair(1, 20), new LongPair(2, 10))); - set.add(Range.closed(new LongPair(2, 25), new LongPair(2, 28))); - set.add(Range.closed(new LongPair(3, 12), new LongPair(3, 20))); - set.add(Range.closed(new LongPair(4, 12), new LongPair(4, 20))); - gSet.add(Range.closed(new LongPair(1, 10), new LongPair(1, 15))); - gSet.add(Range.closed(new LongPair(1, 20), new LongPair(2, 10))); - gSet.add(Range.closed(new LongPair(2, 25), new LongPair(2, 28))); - gSet.add(Range.closed(new LongPair(3, 12), new LongPair(3, 20))); - gSet.add(Range.closed(new LongPair(4, 12), new LongPair(4, 20))); - - LongPair position = new LongPair(0, 99); - assertEquals(set.rangeContaining(position.getKey(), position.getValue()), - Range.closed(new LongPair(0, 98), new LongPair(0, 100))); - assertEquals(set.rangeContaining(position.getKey(), position.getValue()), gSet.rangeContaining(position)); - - position = new LongPair(2, 30); - assertNull(set.rangeContaining(position.getKey(), position.getValue())); - assertEquals(set.rangeContaining(position.getKey(), position.getValue()), gSet.rangeContaining(position)); - - position = new LongPair(3, 13); - assertEquals(set.rangeContaining(position.getKey(), position.getValue()), - Range.closed(new LongPair(3, 12), new LongPair(3, 20))); - assertEquals(set.rangeContaining(position.getKey(), position.getValue()), gSet.rangeContaining(position)); - - position = new LongPair(3, 22); - assertNull(set.rangeContaining(position.getKey(), position.getValue())); - assertEquals(set.rangeContaining(position.getKey(), position.getValue()), gSet.rangeContaining(position)); - } - - - private List> getConnectedRange(Set> gRanges) { - List> gRangeConnected = new ArrayList(); - Range lastRange = null; - for (Range range : gRanges) { - if (lastRange == null) { - lastRange = range; - continue; - } - LongPair previousUpper = lastRange.upperEndpoint(); - LongPair currentLower = range.lowerEndpoint(); - int previousUpperValue = (int) (lastRange.upperBoundType().equals(BoundType.CLOSED) - ? previousUpper.getValue() - : previousUpper.getValue() - 1); - int currentLowerValue = (int) (range.lowerBoundType().equals(BoundType.CLOSED) ? currentLower.getValue() - : currentLower.getValue() + 1); - boolean connected = - previousUpper.getKey() == currentLower.getKey() && (previousUpperValue >= currentLowerValue); - if (connected) { - lastRange = Range.closed(lastRange.lowerEndpoint(), range.upperEndpoint()); - } else { - gRangeConnected.add(lastRange); - lastRange = range; - } - } - int lowerOpenValue = (int) (lastRange.lowerBoundType().equals(BoundType.CLOSED) - ? (lastRange.lowerEndpoint().getValue() - 1) - : lastRange.lowerEndpoint().getValue()); - lastRange = Range.openClosed(new LongPair(lastRange.lowerEndpoint().getKey(), lowerOpenValue), - lastRange.upperEndpoint()); - gRangeConnected.add(lastRange); - return gRangeConnected; - } -} \ No newline at end of file diff --git a/microbench/src/main/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetBenchmark.java b/microbench/src/main/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetBenchmark.java new file mode 100644 index 0000000000000..1e920ef62a4d6 --- /dev/null +++ b/microbench/src/main/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetBenchmark.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.impl; + +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Baseline throughput numbers for {@link PositionRangeSet} on realistic cursor workloads. Hot paths + * exercised: {@code addOpenClosed} (per individual ack), {@code contains} (per message read), + * {@code forEachRawRange} (per persistence flush), {@code removeAtMost} (per mark-delete), and + * {@code toRanges} (serialization for the cursor ledger). + * + *

Run with: + * 

{@code
+ * ./gradlew :microbench:shadowJar
+ * java -jar microbench/build/libs/microbench-*-benchmarks.jar PositionRangeSetBenchmark
+ * }
+ */ +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@State(Scope.Benchmark) +@Warmup(time = 2, iterations = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(time = 3, iterations = 1, timeUnit = TimeUnit.SECONDS) +@Fork(1) +public class PositionRangeSetBenchmark { + + @Param({"10", "100", "1000"}) + public int ledgerCount; + + @Param({"100", "10000"}) + public int entriesPerLedger; + + private PositionRangeSet set; + private final AtomicLong cursor = new AtomicLong(); + private long totalEntries; + + @Setup(Level.Trial) + public void setup() { + set = new PositionRangeSet(PositionFactory::create, false); + // Pre-populate: every other entry in every ledger is "individually deleted" — models the + // typical Shared/Key_Shared ack pattern that produces fragmented deleted-message ranges. + for (int l = 0; l < ledgerCount; l++) { + for (int e = 1; e < entriesPerLedger; e += 2) { + set.addOpenClosed(l, e - 1, l, e); + } + } + totalEntries = (long) ledgerCount * entriesPerLedger; + cursor.set(0); + } + + @Benchmark + @Threads(1) + public boolean addOpenClosedSameLedger() { + // Simulate a new individual ack arriving on an existing ledger. + long l = (cursor.getAndIncrement() % ledgerCount); + long e = entriesPerLedger + (cursor.getAndIncrement() & 0xFF); + set.addOpenClosed(l, e, l, e + 1); + return true; + } + + @Benchmark + @Threads(1) + public boolean containsHit() { + // Read-path check: is this (already-deleted) entry in the set? + long l = (cursor.getAndIncrement() & Long.MAX_VALUE) % ledgerCount; + long e = 1 + 2 * ((cursor.getAndIncrement() & Long.MAX_VALUE) % (entriesPerLedger / 2)); + return set.contains(l, e); + } + + @Benchmark + @Threads(1) + public boolean containsMiss() { + // Read-path check: entry that was NOT individually deleted (still in the backlog). + long l = (cursor.getAndIncrement() & Long.MAX_VALUE) % ledgerCount; + long e = 2 * ((cursor.getAndIncrement() & Long.MAX_VALUE) % (entriesPerLedger / 2)); + return set.contains(l, e); + } + + @Benchmark + @Threads(1) + public int forEachRawRange(Blackhole bh) { + // Persistence path: walk every deleted range once per mark-delete flush. + int[] count = {0}; + set.forEachRawRange((lowerKey, lowerValue, upperKey, upperValue) -> { + count[0]++; + return true; + }); + return count[0]; + } + + @Benchmark + @Threads(1) + public Map toRanges() { + // Serialization: builds the dense long[] map written to the cursor ledger. + return set.toRanges(Integer.MAX_VALUE); + } + + @Benchmark + @Threads(1) + public int size() { + return set.size(); + } +} diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index d3f308ba7bfce..f0ea95c0f56db 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -2481,9 +2481,8 @@ The max allowed delay for delayed delivery (in milliseconds). If the broker rece @FieldContext(category = CATEGORY_STORAGE_ML, doc = "When storing acknowledgement state, choose a more compact serialization format that stores" - + " individual acknowledgements as a bitmap which is serialized to an array of long values.\n\n" - + "NOTE: This setting requires managedLedgerUnackedRangesOpenCacheSetEnabled=true to be effective.") - private boolean managedLedgerPersistIndividualAckAsLongArray = false; + + " individual acknowledgements as a bitmap which is serialized to an array of long values.") + private boolean managedLedgerPersistIndividualAckAsLongArray = true; @FieldContext( category = CATEGORY_STORAGE_ML, @@ -2505,13 +2504,6 @@ The max allowed delay for delayed delivery (in milliseconds). If the broker rece + "If number of unack message range is higher than this limit then broker will persist" + " unacked ranges into bookkeeper to avoid additional data overhead into MetadataStore.") private int managedLedgerMaxUnackedRangesToPersistInMetadataStore = 1000; - @FieldContext( - category = CATEGORY_STORAGE_OFFLOADING, - doc = "When set to true, a BitSet will be used to track acknowledged messages that come after the \"mark " - + "delete position\" for each subscription.\n\nRoaringBitmap is used as a memory efficient BitSet " - + "implementation for the acknowledged messages tracking. Unacknowledged ranges are the message " - + "ranges excluding the acknowledged messages.") - private boolean managedLedgerUnackedRangesOpenCacheSetEnabled = true; @FieldContext( dynamic = true, category = CATEGORY_STORAGE_ML, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index d6d4ac88e4cff..5586c64ad63be 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -2252,8 +2252,6 @@ public CompletableFuture getManagedLedgerConfig(@NonNull To managedLedgerConfig .setAddEntryTimeoutSeconds(serviceConfig.getManagedLedgerAddEntryTimeoutSeconds()); managedLedgerConfig.setMetadataEnsembleSize(serviceConfig.getManagedLedgerDefaultEnsembleSize()); - managedLedgerConfig.setUnackedRangesOpenCacheSetEnabled( - serviceConfig.isManagedLedgerUnackedRangesOpenCacheSetEnabled()); managedLedgerConfig.setMetadataWriteQuorumSize(serviceConfig.getManagedLedgerDefaultWriteQuorum()); managedLedgerConfig.setMetadataAckQuorumSize(serviceConfig.getManagedLedgerDefaultAckQuorum()); managedLedgerConfig diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageRedeliveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageRedeliveryTest.java index 962749fbd49e9..d9fecdb58887d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageRedeliveryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageRedeliveryTest.java @@ -91,7 +91,6 @@ public void testRedelivery(boolean useOpenRangeSet) throws Exception { this.conf.setManagedLedgerMaxEntriesPerLedger(5); this.conf.setManagedLedgerMinLedgerRolloverTimeMinutes(0); - this.conf.setManagedLedgerUnackedRangesOpenCacheSetEnabled(useOpenRangeSet); @Cleanup("shutdownNow") final ScheduledExecutorService executor = Executors.newScheduledThreadPool(20, new DefaultThreadFactory("pulsar")); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java index 7c65441004bfe..774227ed2b351 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java @@ -24,7 +24,9 @@ import java.nio.ByteBuffer; import java.util.concurrent.locks.StampedLock; import java.util.function.LongConsumer; +import org.roaringbitmap.BitSetUtil; import org.roaringbitmap.PeekableIntIterator; +import org.roaringbitmap.RoaringBitmap; import org.roaringbitmap.buffer.MutableRoaringBitmap; /** @@ -160,14 +162,12 @@ public boolean contains(long value) { @Override public boolean contains(long from, long to) { - if (from < 0 || from > MAX_UINT32 || to <= from) { + if (from < 0 || from > MAX_UINT32 || to <= from || to > UINT32_SIZE) { return false; } long stamp = lock.readLock(); try { - // Clamp: contains treats out-of-range `to` as a query past the uint32 end, but - // RoaringBitmap.contains is unreliable when `to` exceeds UINT32_SIZE. - return bitmap.contains(from, Math.min(to, UINT32_SIZE)); + return bitmap.contains(from, to); } finally { lock.unlockRead(stamp); } @@ -193,6 +193,17 @@ public boolean isEmpty() { } } + @Override + public void clear() { + long stamp = lock.writeLock(); + try { + bitmap.clear(); + removesSinceTrim = 0; + } finally { + lock.unlockWrite(stamp); + } + } + @Override public long nextAbsentValue(long from) { if (from < 0 || from > MAX_UINT32) { @@ -206,6 +217,72 @@ public long nextAbsentValue(long from) { } } + @Override + public long nextPresentValue(long from) { + if (from < 0 || from > MAX_UINT32) { + return -1; + } + long stamp = lock.readLock(); + try { + return bitmap.nextValue((int) from); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public long previousAbsentValue(long from) { + if (from < 0) { + return -1; + } + // Clamp to MAX_UINT32 instead of returning -1 for out-of-range input. + // This allows safe usage in PositionRangeSet.lastRange() where + // previousAbsentValue(lastPresentValue()) is called and lastPresentValue + // may be MAX_UINT32. Clamping avoids the need for Math.min() guards at call sites. + if (from > MAX_UINT32) { + from = MAX_UINT32; + } + long stamp = lock.readLock(); + try { + return bitmap.previousAbsentValue((int) from); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public long lastPresentValue() { + long stamp = lock.readLock(); + try { + if (bitmap.isEmpty()) { + return -1; + } + return bitmap.previousValue(-1); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public long rank(long value) { + if (value <= 0) { + return 0; + } + // Clamp to UINT32_SIZE instead of MAX_UINT32 to allow rank(UINT32_SIZE) to return + // the total cardinality. This is needed in PositionRangeSet.cardinality() where + // rank(upperValue + 1) is called and upperValue may be MAX_UINT32. The clamping + // ensures rank(0x100000000L) counts all values in the uint32 range. + if (value > UINT32_SIZE) { + value = UINT32_SIZE; + } + long stamp = lock.readLock(); + try { + return bitmap.rank((int) (value - 1)); + } finally { + lock.unlockRead(stamp); + } + } + @Override public void or(LongBitmap other) { if (other == this) { @@ -322,6 +399,30 @@ public byte[] serialize() { return bytes; } + @Override + public long[] serializeToLongArray() { + long stamp = lock.readLock(); + try { + RoaringBitmap immutable = bitmap.toRoaringBitmap(); + return BitSetUtil.toLongArray(immutable); + } finally { + lock.unlockRead(stamp); + } + } + + @Override + public void deserializeFromLongArray(long[] data) { + long stamp = lock.writeLock(); + try { + bitmap.clear(); + RoaringBitmap rb = BitSetUtil.bitmapOf(data); + bitmap.or(rb.toMutableRoaringBitmap()); + removesSinceTrim = 0; + } finally { + lock.unlockWrite(stamp); + } + } + static ConcurrentRoaringBitmap deserialize(ByteBuf buf) { try { ByteBuffer nioBuffer = buf.nioBuffer(buf.readerIndex(), buf.readableBytes()); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongBitmap.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongBitmap.java index 5176da8c12598..53432eae603dc 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongBitmap.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongBitmap.java @@ -120,6 +120,9 @@ public interface LongBitmap { /** Returns {@code true} if no values are stored. */ boolean isEmpty(); + /** Removes all values from the bitmap. */ + void clear(); + /** * Returns the smallest absent value greater than or equal to {@code from}. * @@ -128,6 +131,57 @@ public interface LongBitmap { */ long nextAbsentValue(long from); + /** + * Returns the smallest present value greater than or equal to {@code from}. + * + * @param from inclusive lower bound + * @return next present value, or {@code -1} if none exists + */ + long nextPresentValue(long from); + + /** + * Returns the largest absent value less than or equal to {@code from}. + * + *

Boundary behavior: If {@code from > MAX_UINT32 (0xFFFFFFFF)}, it is clamped + * to {@code MAX_UINT32}. This allows safe usage in expressions like + * {@code previousAbsentValue(lastValue)} where {@code lastValue} may be at the boundary, + * avoiding the need for {@code Math.min(result, boundary)} checks at call sites. + * + *

Normal usage (entryId within int range) never triggers this clamping; it only + * applies when computing ranges that may include the uint32 upper boundary. + * + * @param from inclusive upper bound + * @return previous absent value, or {@code -1} if none exists + */ + long previousAbsentValue(long from); + + /** + * Returns the last (highest) present value, or {@code -1} if empty. + * + * @return last present value, or {@code -1} if bitmap is empty + */ + long lastPresentValue(); + + /** + * Returns the number of present values strictly less than {@code value}. + * This is the rank of the value in the sorted sequence of present values. + * + *

Useful for computing range cardinality: + * {@code cardinality(from, to) = rank(to) - rank(from)}. + * + *

Boundary behavior: If {@code value > UINT32_SIZE (0x100000000)}, it is clamped + * to {@code UINT32_SIZE}, effectively returning the total cardinality (all values are less + * than a value beyond the supported range). This allows safe usage in expressions like + * {@code rank(upperValue + 1)} where {@code upperValue} may be {@code MAX_UINT32 (0xFFFFFFFF)}. + * + *

Normal usage never triggers this clamping; it only applies when computing ranges + * that include the uint32 upper boundary. + * + * @param value upper bound (exclusive) + * @return count of present values less than {@code value} + */ + long rank(long value); + /** * Adds all values from {@code other} into this bitmap. * @@ -167,4 +221,19 @@ public interface LongBitmap { * Serializes the bitmap into a newly allocated byte array. */ byte[] serialize(); + + /** + * Serializes the bitmap to long[] format compatible with {@link java.util.BitSet#toLongArray()}. + * This format can be restored using {@link #deserializeFromLongArray(long[])}. + * + * @return long array representing the bitmap + */ + long[] serializeToLongArray(); + + /** + * Deserializes a bitmap from long[] format compatible with {@link java.util.BitSet#valueOf(long[])}. + * + * @param data long array in BitSet format + */ + void deserializeFromLongArray(long[] data); } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongBitmaps.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongBitmaps.java index 489502a439d4a..ac7b4fadbedd9 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongBitmaps.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongBitmaps.java @@ -58,4 +58,20 @@ public static LongBitmap create() { public static LongBitmap deserialize(ByteBuf buf) { return ConcurrentRoaringBitmap.deserialize(buf); } + + /** + * Deserializes a LongBitmap from long[] format compatible with {@link java.util.BitSet}. + * + *

Creates a new LongBitmap populated with values from the long array format + * produced by {@link LongBitmap#serializeToLongArray()}. This format is compatible + * with {@link java.util.BitSet#valueOf(long[])}. + * + * @param data long array in BitSet format + * @return a new LongBitmap containing the deserialized values + */ + public static LongBitmap deserializeFromLongArray(long[] data) { + LongBitmap bitmap = create(); + bitmap.deserializeFromLongArray(data); + return bitmap; + } } diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/LongBitmapTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/LongBitmapTest.java index 35bbf9b5f57b0..333dda0b24e4e 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/LongBitmapTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/LongBitmapTest.java @@ -821,4 +821,113 @@ public void testDrainToIsAtomic() { throw new RuntimeException(e); } } -} + + @Test + public void testDeserializeFromLongArrayFactory() { + LongBitmap bitmap = LongBitmaps.create(); + bitmap.add(1); + bitmap.add(100); + bitmap.add(1000); + bitmap.add(Integer.MAX_VALUE); + + long[] data = bitmap.serializeToLongArray(); + + LongBitmap restored = LongBitmaps.deserializeFromLongArray(data); + + assertEquals(restored.cardinality(), 4); + assertTrue(restored.contains(1)); + assertTrue(restored.contains(100)); + assertTrue(restored.contains(1000)); + assertTrue(restored.contains(Integer.MAX_VALUE)); + } + + @Test + public void testLastPresentValueWithMaxUint32() { + LongBitmap bitmap = LongBitmaps.create(); + bitmap.add(0xFFFFFFFFL); + + assertEquals(bitmap.lastPresentValue(), 0xFFFFFFFFL); + + bitmap.add(Integer.MAX_VALUE); + assertEquals(bitmap.lastPresentValue(), 0xFFFFFFFFL); + + bitmap.clear(); + assertEquals(bitmap.lastPresentValue(), -1); + } + + @Test + public void testNavigationWithUint32Boundary() { + LongBitmap bitmap = LongBitmaps.create(); + bitmap.add(Integer.MAX_VALUE); + bitmap.add(Integer.MAX_VALUE + 1L); + bitmap.add(0xFFFFFFFFL); + + assertEquals(bitmap.nextPresentValue(Integer.MAX_VALUE), Integer.MAX_VALUE); + assertEquals(bitmap.nextPresentValue(Integer.MAX_VALUE + 1L), Integer.MAX_VALUE + 1L); + assertEquals(bitmap.nextPresentValue(0xFFFFFFFEL), 0xFFFFFFFFL); + + assertEquals(bitmap.nextAbsentValue(0xFFFFFFFFL), -1); + + assertEquals(bitmap.nextPresentValue(0xFFFFFFFFL), 0xFFFFFFFFL); + assertEquals(bitmap.nextPresentValue(0xFFFFFFFFL + 1L), -1); + } + + @Test + public void testRank() { + LongBitmap bitmap = LongBitmaps.create(); + bitmap.add(10); + bitmap.add(20); + bitmap.add(30); + bitmap.add(40); + bitmap.add(50); + + assertEquals(bitmap.rank(0), 0); + assertEquals(bitmap.rank(10), 0); + assertEquals(bitmap.rank(11), 1); + assertEquals(bitmap.rank(20), 1); + assertEquals(bitmap.rank(25), 2); + assertEquals(bitmap.rank(50), 4); + assertEquals(bitmap.rank(51), 5); + assertEquals(bitmap.rank(100), 5); + + bitmap.add(0xFFFFFFFFL); + + assertEquals(bitmap.rank(0xFFFFFFFFL), 5); + assertEquals(bitmap.rank(0x100000000L), 6); + } + + @Test + public void testPreviousAbsentValue() { + LongBitmap bitmap = LongBitmaps.create(); + bitmap.add(5); + bitmap.add(10); + bitmap.add(15); + bitmap.add(20); + + assertEquals(bitmap.previousAbsentValue(25), 25); + assertEquals(bitmap.previousAbsentValue(21), 21); + assertEquals(bitmap.previousAbsentValue(20), 19); + assertEquals(bitmap.previousAbsentValue(16), 16); + assertEquals(bitmap.previousAbsentValue(15), 14); + assertEquals(bitmap.previousAbsentValue(6), 6); + assertEquals(bitmap.previousAbsentValue(5), 4); + assertEquals(bitmap.previousAbsentValue(2), 2); + + bitmap.clear(); + bitmap.add(0xFFFFFFFFL); + + assertEquals(bitmap.previousAbsentValue(0xFFFFFFFFL), 0xFFFFFFFEL); + assertEquals(bitmap.previousAbsentValue(0x100000000L), 0xFFFFFFFEL); + } + + @Test + public void testNextPresentValueAtMaxUint32() { + LongBitmap bitmap = LongBitmaps.create(); + + bitmap.add(0xFFFFFFFFL); + + assertEquals(bitmap.nextPresentValue(0), 0xFFFFFFFFL); + assertEquals(bitmap.nextPresentValue(0xFFFFFFFFL), 0xFFFFFFFFL); + assertEquals(bitmap.nextPresentValue(0x100000000L), -1); + } +} \ No newline at end of file From a3c92edad4a4f3f3a7eb522023b46de8e39dd71a Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Thu, 27 Aug 2026 17:50:54 +0800 Subject: [PATCH 196/213] Keep managedLedgerPersistIndividualAckAsLongArray default at false #26127 flips this default to true upstream. Deliberately keep the fork's existing default (false) to preserve upgrade compatibility for existing clusters; the new PositionRangeSet-backed format remains opt-in. Deviation from apache/master recorded during the 2026-08-27 alignment (PRs #25384/#26010/#26117/#26127). --- conf/broker.conf | 2 +- conf/standalone.conf | 2 +- .../java/org/apache/pulsar/broker/ServiceConfiguration.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index 69721610c7feb..667e10f11f122 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -1380,7 +1380,7 @@ managedLedgerMaxBatchDeletedIndexToPersist=10000 # When storing acknowledgement state, choose a more compact serialization format that stores # individual acknowledgements as a bitmap which is serialized to an array of long values. -managedLedgerPersistIndividualAckAsLongArray=true +managedLedgerPersistIndividualAckAsLongArray=false # Max number of "acknowledgment holes" that can be stored in MetadataStore. If number of unack message range is higher # than this limit then broker will persist unacked ranges into bookkeeper to avoid additional data overhead into diff --git a/conf/standalone.conf b/conf/standalone.conf index 0a50c41d44b85..87f7fa904eb90 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -876,7 +876,7 @@ managedLedgerMaxBatchDeletedIndexToPersist=10000 # When storing acknowledgement state, choose a more compact serialization format that stores # individual acknowledgements as a bitmap which is serialized to an array of long values. -managedLedgerPersistIndividualAckAsLongArray=true +managedLedgerPersistIndividualAckAsLongArray=false # Max number of "acknowledgment holes" that can be stored in MetadataStore. If number of unack message range is higher # than this limit then broker will persist unacked ranges into bookkeeper to avoid additional data overhead into diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index f0ea95c0f56db..371725caff3db 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -2482,7 +2482,7 @@ The max allowed delay for delayed delivery (in milliseconds). If the broker rece @FieldContext(category = CATEGORY_STORAGE_ML, doc = "When storing acknowledgement state, choose a more compact serialization format that stores" + " individual acknowledgements as a bitmap which is serialized to an array of long values.") - private boolean managedLedgerPersistIndividualAckAsLongArray = true; + private boolean managedLedgerPersistIndividualAckAsLongArray = false; @FieldContext( category = CATEGORY_STORAGE_ML, From 33380cc9e1ee4e4c106ff9afabef61ba18c0222f Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Thu, 27 Aug 2026 17:56:26 +0800 Subject: [PATCH 197/213] [fix][broker] Remove BucketDelayedDeliveryTrackerBenchmark Signed-off-by: Zixuan Liu --- ...BucketDelayedDeliveryTrackerBenchmark.java | 321 ------------------ .../bucket/MockBucketSnapshotStorage.java | 106 ------ .../broker/delayed/bucket/package-info.java | 27 -- 3 files changed, 454 deletions(-) delete mode 100644 microbench/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerBenchmark.java delete mode 100644 microbench/src/main/java/org/apache/pulsar/broker/delayed/bucket/MockBucketSnapshotStorage.java delete mode 100644 microbench/src/main/java/org/apache/pulsar/broker/delayed/bucket/package-info.java diff --git a/microbench/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerBenchmark.java b/microbench/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerBenchmark.java deleted file mode 100644 index 08d02195c8781..0000000000000 --- a/microbench/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerBenchmark.java +++ /dev/null @@ -1,321 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.delayed.bucket; - -import io.netty.util.HashedWheelTimer; -import io.netty.util.Timer; -import io.netty.util.concurrent.DefaultThreadFactory; -import java.time.Clock; -import java.util.NavigableSet; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.PositionFactory; -import org.apache.bookkeeper.mledger.impl.ActiveManagedCursorContainerImpl; -import org.apache.bookkeeper.mledger.impl.MockManagedCursor; -import org.apache.pulsar.broker.delayed.DelayedDeliveryTracker; -import org.apache.pulsar.broker.delayed.NoopDelayedDeliveryContext; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Level; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Param; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.TearDown; -import org.openjdk.jmh.annotations.Threads; -import org.openjdk.jmh.annotations.Warmup; - -/** - * JMH benchmarks for {@link BucketDelayedDeliveryTracker}. - * - *

This benchmark measures tracker throughput under different read/write ratios - * and initial message counts without implying a specific lock implementation. - * - *

Run with: mvn exec:java -Dexec.mainClass="org.openjdk.jmh.Main" - * -Dexec.args="BucketDelayedDeliveryTrackerBenchmark" - */ -@BenchmarkMode(Mode.Throughput) -@OutputTimeUnit(TimeUnit.SECONDS) -@State(Scope.Benchmark) -@Warmup(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1) -@Measurement(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1) -@Fork(1) -public class BucketDelayedDeliveryTrackerBenchmark { - - /** - * Fixed delivery timestamp base that stays far beyond any benchmark trial duration, - * so scheduled tasks will not start firing while throughput is being measured. - */ - private static final long FUTURE_DELIVERY_BASE_TIME_MILLIS = 4102444800000L; // 2100-01-01T00:00:00Z - - @Param({"90_10", "80_20", "70_30", "50_50"}) - public String readWriteRatio; - - @Param({"1000", "5000", "8000"}) - public int initialMessages; - - private BucketDelayedDeliveryTracker tracker; - private Timer timer; - private MockBucketSnapshotStorage storage; - private NoopDelayedDeliveryContext context; - private AtomicLong messageIdGenerator; - private int readPercentage; - private long futureDeliveryBaseTimeMillis; - /** - * Maximum number of additional unique (ledgerId, entryId) positions to - * introduce per trial on top of {@link #initialMessages}. This allows - * controlling the memory footprint of the benchmark while still applying - * sustained write pressure to the tracker. - * - *

Use {@code -p maxAdditionalUniqueMessages=...} on the JMH command line - * to tune the load. The default value is conservative for local runs.

- */ - @Param({"1000000"}) - public long maxAdditionalUniqueMessages; - /** - * Upper bound on the absolute message id that will be used to derive - * (ledgerId, entryId) positions during a single trial. - */ - private long maxUniqueMessageId; - /** - * In real Pulsar usage, {@link DelayedDeliveryTracker#addMessage(long, long, long)} is invoked - * by a single dispatcher thread and messages arrive in order of (ledgerId, entryId). - *

- * To reflect this invariant in the benchmark, all write operations that end up calling - * {@code tracker.addMessage(...)} are serialized via this mutex so that the tracker only - * ever observes a single writer with monotonically increasing ids, even when JMH runs the - * benchmark method with multiple threads. - */ - private final Object writeMutex = new Object(); - - @Setup(Level.Trial) - public void setup() throws Exception { - setupMockComponents(); - createTracker(); - String[] parts = readWriteRatio.split("_"); - readPercentage = Integer.parseInt(parts[0]); - futureDeliveryBaseTimeMillis = FUTURE_DELIVERY_BASE_TIME_MILLIS; - preloadMessages(); - messageIdGenerator = new AtomicLong(initialMessages + 1); - // Allow a bounded number of additional unique messages per trial to avoid - // unbounded memory growth while still stressing the indexing logic. - maxUniqueMessageId = initialMessages + maxAdditionalUniqueMessages; - } - - @TearDown(Level.Trial) - public void tearDown() throws Exception { - if (tracker != null) { - tracker.close(); - } - if (timer != null) { - timer.stop(); - } - } - - private void setupMockComponents() throws Exception { - timer = new HashedWheelTimer(new DefaultThreadFactory("test-delayed-delivery"), 100, TimeUnit.MILLISECONDS); - storage = new MockBucketSnapshotStorage(); - - ActiveManagedCursorContainerImpl container = new ActiveManagedCursorContainerImpl(); - MockManagedCursor cursor = MockManagedCursor.createCursor(container, "test-cursor", - PositionFactory.create(0, 0)); - // Use the same " / " naming pattern as real dispatchers, - // so that Bucket.asyncSaveBucketSnapshot can correctly derive topicName. - String dispatcherName = "persistent://public/default/jmh-topic / " + cursor.getName(); - context = new NoopDelayedDeliveryContext(dispatcherName, cursor); - } - - private void createTracker() throws Exception { - tracker = new BucketDelayedDeliveryTracker( - context, timer, 1000, Clock.systemUTC(), true, storage, - 20, 1000, 100, 50 - ); - } - - private void preloadMessages() { - // Preload messages to create realistic test conditions while keeping - // delivery timestamps far beyond the benchmark trial duration so the - // tracker's timer does not start firing during measurement. - long baseTime = futureDeliveryBaseTimeMillis; - for (int i = 1; i <= initialMessages; i++) { - tracker.addMessage(i, i, baseTime + i * 1000L); - } - } - - // ============================================================================= - // READ-WRITE RATIO BENCHMARKS - // ============================================================================= - - @Benchmark - public boolean benchmarkMixedOperations() { - if (ThreadLocalRandom.current().nextInt(100) < readPercentage) { - // Read operations - return performReadOperation(); - } else { - // Write operations - return performWriteOperation(); - } - } - - /** - * Serialize calls to {@link BucketDelayedDeliveryTracker#addMessage(long, long, long)} and - * ensure (ledgerId, entryId) are generated in a strictly increasing sequence, matching the - * real dispatcher single-threaded behaviour. - */ - private boolean addMessageSequential(long deliverAt, int entryIdModulo) { - synchronized (writeMutex) { - long id = messageIdGenerator.getAndIncrement(); - // Limit the number of distinct positions that are introduced into the tracker - // to keep memory usage bounded. Once the upper bound is reached, we re-use - // the last position id so that subsequent calls behave like updates to - // existing messages and are short-circuited by containsMessage checks. - long boundedId = Math.min(id, maxUniqueMessageId); - long ledgerId = boundedId; - long entryId = boundedId % entryIdModulo; - return tracker.addMessage(ledgerId, entryId, deliverAt); - } - } - - private boolean performReadOperation() { - int operation = ThreadLocalRandom.current().nextInt(3); - switch (operation) { - case 0: - // containsMessage - long ledgerId = ThreadLocalRandom.current().nextLong(1, initialMessages + 100); - long entryId = ThreadLocalRandom.current().nextLong(1, 1000); - return tracker.containsMessage(ledgerId, entryId); - case 1: - // nextDeliveryTime - try { - tracker.nextDeliveryTime(); - return true; - } catch (Exception e) { - return false; - } - case 2: - // getNumberOfDelayedMessages - long count = tracker.getNumberOfDelayedMessages(); - return count >= 0; - default: - return false; - } - } - - private boolean performWriteOperation() { - long deliverAt = futureDeliveryBaseTimeMillis + ThreadLocalRandom.current().nextLong(5000, 30000); - return addMessageSequential(deliverAt, 1000); - } - - // ============================================================================= - // SPECIFIC OPERATION BENCHMARKS - // ============================================================================= - - @Benchmark - @Threads(8) - public boolean benchmarkConcurrentContainsMessage() { - long ledgerId = ThreadLocalRandom.current().nextLong(1, initialMessages + 100); - long entryId = ThreadLocalRandom.current().nextLong(1, 1000); - return tracker.containsMessage(ledgerId, entryId); - } - - @Benchmark - @Threads(4) - public boolean benchmarkConcurrentAddMessage() { - long deliverAt = futureDeliveryBaseTimeMillis + ThreadLocalRandom.current().nextLong(10000, 60000); - return addMessageSequential(deliverAt, 1000); - } - - @Benchmark - @Threads(2) - public NavigableSet benchmarkConcurrentGetScheduledMessages() { - // Create some messages ready for delivery - long currentTime = System.currentTimeMillis(); - for (int i = 0; i < 5; i++) { - addMessageSequential(currentTime - 1000, 100); - } - return tracker.getScheduledMessages(10); - } - - @Benchmark - @Threads(16) - public long benchmarkConcurrentNextDeliveryTime() { - try { - return tracker.nextDeliveryTime(); - } catch (Exception e) { - return -1; - } - } - - @Benchmark - @Threads(1) - public long benchmarkGetNumberOfDelayedMessages() { - return tracker.getNumberOfDelayedMessages(); - } - - // ============================================================================= - // HIGH CONTENTION SCENARIOS - // ============================================================================= - - @Benchmark - @Threads(32) - public boolean benchmarkHighContentionMixedOperations() { - return benchmarkMixedOperations(); - } - - @Benchmark - @Threads(16) - public boolean benchmarkContentionReads() { - return performReadOperation(); - } - - @Benchmark - @Threads(8) - public boolean benchmarkContentionWrites() { - return performWriteOperation(); - } - - // ============================================================================= - // THROUGHPUT BENCHMARKS - // ============================================================================= - - @Benchmark - @Threads(1) - public boolean benchmarkSingleThreadedThroughput() { - return benchmarkMixedOperations(); - } - - @Benchmark - @Threads(4) - public boolean benchmarkMediumConcurrencyThroughput() { - return benchmarkMixedOperations(); - } - - @Benchmark - @Threads(8) - public boolean benchmarkHighConcurrencyThroughput() { - return benchmarkMixedOperations(); - } - -} diff --git a/microbench/src/main/java/org/apache/pulsar/broker/delayed/bucket/MockBucketSnapshotStorage.java b/microbench/src/main/java/org/apache/pulsar/broker/delayed/bucket/MockBucketSnapshotStorage.java deleted file mode 100644 index c89071d31a097..0000000000000 --- a/microbench/src/main/java/org/apache/pulsar/broker/delayed/bucket/MockBucketSnapshotStorage.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.delayed.bucket; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicLong; -import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; -import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; - -public class MockBucketSnapshotStorage implements BucketSnapshotStorage { - - private final AtomicLong idGenerator = new AtomicLong(1); - private final Map snapshots = new ConcurrentHashMap<>(); - private final Map> snapshotSegments = new ConcurrentHashMap<>(); - private final Map snapshotLengths = new ConcurrentHashMap<>(); - - @Override - public CompletableFuture createBucketSnapshot(SnapshotMetadata snapshotMetadata, - List bucketSnapshotSegments, - String bucketKey, String topicName, String cursorName) { - long id = idGenerator.getAndIncrement(); - snapshots.put(id, snapshotMetadata); - snapshotSegments.put(id, new ArrayList<>(bucketSnapshotSegments)); - long snapshotLength = snapshotMetadata.toByteArray().length; - for (SnapshotSegment bucketSnapshotSegment : bucketSnapshotSegments) { - snapshotLength += bucketSnapshotSegment.toByteArray().length; - } - snapshotLengths.put(id, snapshotLength); - return CompletableFuture.completedFuture(id); - } - - @Override - public CompletableFuture getBucketSnapshotMetadata(long bucketId) { - SnapshotMetadata metadata = snapshots.get(bucketId); - return CompletableFuture.completedFuture(metadata); - } - - @Override - public CompletableFuture> getBucketSnapshotSegment(long bucketId, - long firstSegmentEntryId, - long lastSegmentEntryId) { - List segments = snapshotSegments.get(bucketId); - if (segments == null) { - return CompletableFuture.failedFuture( - new IllegalArgumentException("Bucket snapshot segments not found: " + bucketId)); - } - if (firstSegmentEntryId > lastSegmentEntryId) { - return CompletableFuture.completedFuture(Collections.emptyList()); - } - - int fromIndex = Math.toIntExact(firstSegmentEntryId - 1); - int toIndex = Math.toIntExact(lastSegmentEntryId); - if (fromIndex < 0 || fromIndex >= segments.size()) { - return CompletableFuture.failedFuture( - new IllegalArgumentException("Invalid first segment entry id: " + firstSegmentEntryId)); - } - toIndex = Math.min(toIndex, segments.size()); - return CompletableFuture.completedFuture(new ArrayList<>(segments.subList(fromIndex, toIndex))); - } - - @Override - public CompletableFuture getBucketSnapshotLength(long bucketId) { - return CompletableFuture.completedFuture(snapshotLengths.getOrDefault(bucketId, 0L)); - } - - @Override - public CompletableFuture deleteBucketSnapshot(long bucketId) { - snapshots.remove(bucketId); - snapshotSegments.remove(bucketId); - snapshotLengths.remove(bucketId); - return CompletableFuture.completedFuture(null); - } - - @Override - public void start() throws Exception { - // No-op - } - - @Override - public void close() throws Exception { - snapshots.clear(); - snapshotSegments.clear(); - snapshotLengths.clear(); - } -} diff --git a/microbench/src/main/java/org/apache/pulsar/broker/delayed/bucket/package-info.java b/microbench/src/main/java/org/apache/pulsar/broker/delayed/bucket/package-info.java deleted file mode 100644 index 188ce0be224cd..0000000000000 --- a/microbench/src/main/java/org/apache/pulsar/broker/delayed/bucket/package-info.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/** - * Microbenchmarks for delayed message delivery bucket implementation. - * - *

This package contains JMH benchmarks for testing the performance - * characteristics of the BucketDelayedDeliveryTracker, particularly - * focusing on thread safety improvements with StampedLock optimistic reads. - */ -package org.apache.pulsar.broker.delayed.bucket; \ No newline at end of file From d0786183155901ae71a9ec422feaff8cd1b1336f Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Mon, 20 Jul 2026 16:57:03 +0800 Subject: [PATCH 198/213] [improve][broker] Optimize SegmentedLongArray with heap-backed long[][] (#26183) (cherry picked from commit 060b1306dc88f0c8f1f07a6ec7427026957daca6) --- .../util/collections/SegmentedLongArray.java | 266 ++++++++++--- .../collections/TripleLongPriorityQueue.java | 6 +- .../collections/SegmentedLongArrayTest.java | 358 +++++++++++++++++- 3 files changed, 555 insertions(+), 75 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/SegmentedLongArray.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/SegmentedLongArray.java index c551895c51a92..f796ccf0c1239 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/SegmentedLongArray.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/SegmentedLongArray.java @@ -18,111 +18,259 @@ */ package org.apache.pulsar.common.util.collections; -import io.netty.buffer.ByteBuf; -import java.util.ArrayList; -import java.util.List; +import static com.google.common.base.Preconditions.checkArgument; +import com.google.common.annotations.VisibleForTesting; +import java.util.Arrays; import javax.annotation.concurrent.NotThreadSafe; import lombok.Getter; -import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; +/** + * A growable array of {@code long} values backed by heap-allocated segments. + * + *

This class provides a logical contiguous {@code long[]} whose size may exceed + * {@link Integer#MAX_VALUE}. Internally, the storage is split into fixed-size + * segments, allowing capacities larger than a single Java array while keeping + * element access in constant time. + * + *

Segment layout invariant: + *

    + *
  • Every segment except the last has length {@code segmentSize}.
    • + *
  • The last segment may be partially filled.
    • + *
  • {@code capacity} always equals the total number of allocated elements + * across all segments.
    • + *
+ * + *

The segment invariant guarantees that the bit based address mapping + * ({@code offset >>> segmentShift}, {@code offset & segmentMask}) remains + * valid for every logical offset. + * + *

Growing and shrinking preserve existing contents while maintaining the + * segment layout invariant. + * + *

This class is not thread-safe. + */ @NotThreadSafe public class SegmentedLongArray implements AutoCloseable { - private static final int SIZE_OF_LONG = 8; + /** + * Default number of {@code long} values in a full segment. Production behavior + * is fixed by this constant; tests may override it via the + * {@link #SegmentedLongArray(long, int) package-private constructor}. + */ + static final int DEFAULT_SEGMENT_SIZE = 2 * 1024 * 1024; + + static { + assert Integer.bitCount(DEFAULT_SEGMENT_SIZE) == 1 + : "DEFAULT_SEGMENT_SIZE must be a power of 2"; + } + + private final int segmentSize; + private final int segmentShift; + private final int segmentMask; - private static final int MAX_SEGMENT_SIZE = 2 * 1024 * 1024; // 2M longs -> 16 MB - private final List buffers = new ArrayList<>(); + private long[][] segments; + private int segmentCount; + /** Minimum capacity to which this array may be shrunk. */ @Getter private final long initialCapacity; + /** Logical capacity, measured in {@code long} elements. */ @Getter private long capacity; + /** Total bytes allocated by all live backing segments. */ + private long allocatedBytes; + + /** + * Creates a segmented array with the specified initial capacity and the default + * segment size ({@link #DEFAULT_SEGMENT_SIZE}). + * + * @param initialCapacity initial capacity in {@code long} elements + * @throws IllegalArgumentException if {@code initialCapacity <= 0} + */ public SegmentedLongArray(long initialCapacity) { - long remainingToAdd = initialCapacity; - - // Add first segment - int sizeToAdd = (int) Math.min(remainingToAdd, MAX_SEGMENT_SIZE); - ByteBuf buffer = PulsarByteBufAllocator.DEFAULT.directBuffer(sizeToAdd * SIZE_OF_LONG); - buffer.writerIndex(sizeToAdd * SIZE_OF_LONG); - buffers.add(buffer); - remainingToAdd -= sizeToAdd; - - // Add the remaining segments, all at full segment size, if necessary - while (remainingToAdd > 0) { - buffer = PulsarByteBufAllocator.DEFAULT.directBuffer(MAX_SEGMENT_SIZE * SIZE_OF_LONG); - buffer.writerIndex(MAX_SEGMENT_SIZE * SIZE_OF_LONG); - buffers.add(buffer); - remainingToAdd -= MAX_SEGMENT_SIZE; - } + this(initialCapacity, DEFAULT_SEGMENT_SIZE); + } + /** + * Creates a segmented array with a custom segment size. + * + *

Intended for unit tests that exercise multi-segment grow/shrink behavior + * without allocating production-sized (16 MiB) backing arrays. Production + * callers should use {@link #SegmentedLongArray(long)}. + * + * @param initialCapacity initial capacity in {@code long} elements + * @param segmentSize number of {@code long} values per full segment; must be + * a positive power of two + * @throws IllegalArgumentException if {@code initialCapacity <= 0} or + * {@code segmentSize} is not a positive power of two + */ + @VisibleForTesting + SegmentedLongArray(long initialCapacity, int segmentSize) { + checkArgument(initialCapacity > 0, "initialCapacity must be positive"); + checkArgument(segmentSize > 0 && Integer.bitCount(segmentSize) == 1, + "segmentSize must be a positive power of two"); + this.segmentSize = segmentSize; + this.segmentShift = Integer.numberOfTrailingZeros(segmentSize); + this.segmentMask = segmentSize - 1; this.initialCapacity = initialCapacity; - this.capacity = this.initialCapacity; + this.capacity = initialCapacity; + allocateSegments(initialCapacity); + } + + /** + * Allocates the initial segment layout. + */ + private void allocateSegments(long longCapacity) { + segmentCount = Math.max(1, (int) ((longCapacity + segmentSize - 1) / segmentSize)); + segments = new long[segmentCount][]; + + long remaining = longCapacity; + long bytes = 0; + + for (int i = 0; i < segmentCount; i++) { + int size = (int) Math.min(segmentSize, remaining); + segments[i] = new long[size]; + bytes += (long) size * Long.BYTES; + remaining -= size; + } + + allocatedBytes = bytes; } public void writeLong(long offset, long value) { - int bufferIdx = (int) (offset / MAX_SEGMENT_SIZE); - int internalIdx = (int) (offset % MAX_SEGMENT_SIZE); - buffers.get(bufferIdx).setLong(internalIdx * SIZE_OF_LONG, value); + long[] segment = segments[(int) (offset >>> segmentShift)]; + segment[(int) (offset & segmentMask)] = value; } public long readLong(long offset) { - int bufferIdx = (int) (offset / MAX_SEGMENT_SIZE); - int internalIdx = (int) (offset % MAX_SEGMENT_SIZE); - return buffers.get(bufferIdx).getLong(internalIdx * SIZE_OF_LONG); + long[] segment = segments[(int) (offset >>> segmentShift)]; + return segment[(int) (offset & segmentMask)]; + } + + /** + * Ensures that the backing storage can hold at least {@code required} + * elements. + * + * @param required minimum required capacity in {@code long} elements + */ + public void ensureCapacity(long required) { + if (required <= capacity) { + return; + } + + long geometric; + if (capacity < segmentSize) { + geometric = Math.min( + capacity + (capacity <= 256 ? capacity : capacity / 2), + segmentSize); + } else { + geometric = capacity + segmentSize; + } + + growTo(Math.max(required, geometric)); } public void increaseCapacity() { - if (capacity < MAX_SEGMENT_SIZE) { - // Resize the current buffer to bigger capacity - capacity += (capacity <= 256 ? capacity : capacity / 2); - capacity = Math.min(capacity, MAX_SEGMENT_SIZE); - buffers.get(0).capacity((int) this.capacity * SIZE_OF_LONG); - buffers.get(0).writerIndex((int) this.capacity * SIZE_OF_LONG); + ensureCapacity(capacity + 1); + } + + /** + * Expands the backing storage to exactly {@code newCapacity}. + */ + private void growTo(long newCapacity) { + if (newCapacity <= capacity) { + return; + } + + int newSegmentCount = (int) ((newCapacity + segmentSize - 1) / segmentSize); + + if (segments.length < newSegmentCount) { + segments = Arrays.copyOf(segments, newSegmentCount); + } + + // If the current last segment becomes an interior segment, + // it must be expanded to preserve the bit-based address mapping. + if (newSegmentCount > segmentCount && segmentCount >= 1) { + int oldLastIdx = segmentCount - 1; + if (segments[oldLastIdx].length < segmentSize) { + resizeLastSegment(oldLastIdx, segmentSize); + } + } + + for (int i = segmentCount; i < newSegmentCount - 1; i++) { + segments[i] = new long[segmentSize]; + allocatedBytes += (long) segmentSize * Long.BYTES; + } + + int newLastIdx = newSegmentCount - 1; + int newLastSize = (int) (newCapacity - (long) newLastIdx * segmentSize); + + if (newLastIdx >= segmentCount) { + segments[newLastIdx] = new long[newLastSize]; + allocatedBytes += (long) newLastSize * Long.BYTES; } else { - // Let's add 1 mode buffer to the list - int bufferSize = MAX_SEGMENT_SIZE * SIZE_OF_LONG; - ByteBuf buffer = PulsarByteBufAllocator.DEFAULT.directBuffer(bufferSize, bufferSize); - buffer.writerIndex(bufferSize); - buffers.add(buffer); - capacity += MAX_SEGMENT_SIZE; + resizeLastSegment(newLastIdx, newLastSize); } + + segmentCount = newSegmentCount; + capacity = newCapacity; } + private void resizeLastSegment(int idx, int newSize) { + long[] old = segments[idx]; + if (old.length == newSize) { + return; + } + + allocatedBytes += (long) (newSize - old.length) * Long.BYTES; + segments[idx] = Arrays.copyOf(old, newSize); + } + + /** + * Shrinks the backing storage to {@code newCapacity}. + * + * @param newCapacity target capacity in {@code long} elements + */ public void shrink(long newCapacity) { if (newCapacity >= capacity || newCapacity < initialCapacity) { return; } - long sizeToReduce = capacity - newCapacity; - while (sizeToReduce >= MAX_SEGMENT_SIZE && buffers.size() > 1) { - ByteBuf b = buffers.remove(buffers.size() - 1); - b.release(); - capacity -= MAX_SEGMENT_SIZE; - sizeToReduce -= MAX_SEGMENT_SIZE; + int newSegmentCount = (int) ((newCapacity + segmentSize - 1) / segmentSize); + int newLastIdx = newSegmentCount - 1; + int newLastSize = (int) (newCapacity - (long) newLastIdx * segmentSize); + + for (int i = newSegmentCount; i < segmentCount; i++) { + allocatedBytes -= (long) segments[i].length * Long.BYTES; + segments[i] = null; } - if (buffers.size() == 1 && sizeToReduce > 0) { - // We should also reduce the capacity of the first buffer - capacity -= sizeToReduce; - ByteBuf oldBuffer = buffers.get(0); - ByteBuf newBuffer = PulsarByteBufAllocator.DEFAULT.directBuffer((int) capacity * SIZE_OF_LONG); - oldBuffer.getBytes(0, newBuffer, (int) capacity * SIZE_OF_LONG); - oldBuffer.release(); - buffers.set(0, newBuffer); + resizeLastSegment(newLastIdx, newLastSize); + + segmentCount = newSegmentCount; + capacity = newCapacity; + + if (segments.length > Math.max(segmentCount * 2L, 16)) { + segments = Arrays.copyOf(segments, segmentCount); } } @Override public void close() { - buffers.forEach(ByteBuf::release); + segments = null; + segmentCount = 0; + capacity = 0; + allocatedBytes = 0; } /** - * The amount of memory used to back the array of longs. + * Returns the physical heap memory reserved by the backing arrays. + * + * @return allocated bytes occupied by all backing segments */ public long bytesCapacity() { - return capacity * SIZE_OF_LONG; + return allocatedBytes; } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueue.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueue.java index cd878c6428459..97fb3b2460d63 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueue.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/TripleLongPriorityQueue.java @@ -23,7 +23,7 @@ /** * Provides a priority-queue implementation specialized on items composed by 3 longs. * - *

This class is not thread safe and the items are stored in direct memory. + *

This class is not thread safe. * *

Algorithm

* @@ -109,9 +109,7 @@ public void close() { */ public void add(long n1, long n2, long n3) { long arrayIdx = tuplesCount * ITEMS_COUNT; - if ((arrayIdx + 2) >= array.getCapacity()) { - array.increaseCapacity(); - } + array.ensureCapacity(arrayIdx + ITEMS_COUNT); siftUp(tuplesCount, n1, n2, n3); ++tuplesCount; diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/SegmentedLongArrayTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/SegmentedLongArrayTest.java index f6c216c439c21..ac41dd0be79e1 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/SegmentedLongArrayTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/SegmentedLongArrayTest.java @@ -19,12 +19,16 @@ package org.apache.pulsar.common.util.collections; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.fail; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; import lombok.Cleanup; import org.testng.annotations.Test; public class SegmentedLongArrayTest { + /** Small segment size so tests can exercise many-segment behavior without 16 MiB allocations. */ + private static final int TEST_SEGMENT_SIZE = 1024; + @Test public void testArray() { @Cleanup @@ -38,15 +42,9 @@ public void testArray() { a.writeLong(2, 2); a.writeLong(3, Long.MAX_VALUE); - try { - a.writeLong(4, Long.MIN_VALUE); - fail("should have failed"); - } catch (IndexOutOfBoundsException e) { - // Expected - } + assertThrows(IndexOutOfBoundsException.class, () -> a.writeLong(4, Long.MIN_VALUE)); a.increaseCapacity(); - a.writeLong(4, Long.MIN_VALUE); assertEquals(a.getCapacity(), 8); @@ -67,10 +65,10 @@ public void testArray() { @Test public void testLargeArray() { - long initialCap = 3 * 1024 * 1024; + long initialCap = TEST_SEGMENT_SIZE + TEST_SEGMENT_SIZE / 2; @Cleanup - SegmentedLongArray a = new SegmentedLongArray(initialCap); + SegmentedLongArray a = new SegmentedLongArray(initialCap, TEST_SEGMENT_SIZE); assertEquals(a.getCapacity(), initialCap); assertEquals(a.bytesCapacity(), initialCap * 8); assertEquals(a.getInitialCapacity(), initialCap); @@ -85,8 +83,9 @@ public void testLargeArray() { a.increaseCapacity(); - assertEquals(a.getCapacity(), 5 * 1024 * 1024); - assertEquals(a.bytesCapacity(), 5 * 1024 * 1024 * 8); + long expectedCap = initialCap + TEST_SEGMENT_SIZE; + assertEquals(a.getCapacity(), expectedCap); + assertEquals(a.bytesCapacity(), expectedCap * 8); assertEquals(a.getInitialCapacity(), initialCap); assertEquals(a.readLong(baseOffset), 0); @@ -100,4 +99,339 @@ public void testLargeArray() { assertEquals(a.bytesCapacity(), initialCap * 8); assertEquals(a.getInitialCapacity(), initialCap); } + + @Test + public void testIncreaseCapacityGrowthPattern() { + @Cleanup + SegmentedLongArray a = new SegmentedLongArray(4); + + a.increaseCapacity(); + assertEquals(a.getCapacity(), 8); + a.increaseCapacity(); + assertEquals(a.getCapacity(), 16); + a.increaseCapacity(); + assertEquals(a.getCapacity(), 32); + } + + @Test + public void testIncreaseCapacityReachesSegmentBoundary() { + long start = TEST_SEGMENT_SIZE - 100; + @Cleanup + SegmentedLongArray a = new SegmentedLongArray(start, TEST_SEGMENT_SIZE); + assertEquals(a.getCapacity(), start); + + a.increaseCapacity(); + assertEquals(a.getCapacity(), TEST_SEGMENT_SIZE); + + a.increaseCapacity(); + assertEquals(a.getCapacity(), TEST_SEGMENT_SIZE * 2L); + } + + @Test + public void testMultiSegmentIncreaseCapacity() { + long initialCap = TEST_SEGMENT_SIZE * 3; + @Cleanup + SegmentedLongArray a = new SegmentedLongArray(initialCap, TEST_SEGMENT_SIZE); + + for (int i = 0; i < 20; i++) { + a.increaseCapacity(); + } + + long expectedCap = TEST_SEGMENT_SIZE * 23L; + assertEquals(a.getCapacity(), expectedCap); + + for (int i = 0; i < 23; i++) { + long offset = (long) i * TEST_SEGMENT_SIZE + 42; + a.writeLong(offset, i); + assertEquals(a.readLong(offset), i); + } + } + + @Test + public void testShrinkDropsWholeSegments() { + long segSize = TEST_SEGMENT_SIZE; + @Cleanup + SegmentedLongArray a = new SegmentedLongArray(segSize, TEST_SEGMENT_SIZE); + for (int i = 0; i < 4; i++) { + a.increaseCapacity(); + } + assertEquals(a.getCapacity(), segSize * 5); + + for (int i = 0; i < 5; i++) { + a.writeLong((long) i * segSize, 100L + i); + } + + a.shrink(segSize * 3); + assertEquals(a.getCapacity(), segSize * 3); + + for (int i = 0; i < 3; i++) { + assertEquals(a.readLong((long) i * segSize), 100L + i); + } + } + + @Test + public void testShrinkToInitialCapacity() { + @Cleanup + SegmentedLongArray a = new SegmentedLongArray(4); + a.increaseCapacity(); + a.increaseCapacity(); + + a.shrink(4); + assertEquals(a.getCapacity(), 4); + assertEquals(a.getInitialCapacity(), 4); + } + + @Test + public void testShrinkBelowInitialFails() { + @Cleanup + SegmentedLongArray a = new SegmentedLongArray(100); + a.shrink(50); + assertEquals(a.getCapacity(), 100); + } + + @Test + public void testSegmentBoundaryReadWrite() { + long segSize = TEST_SEGMENT_SIZE; + @Cleanup + SegmentedLongArray a = new SegmentedLongArray(segSize * 2, TEST_SEGMENT_SIZE); + + a.writeLong(segSize - 1, 111L); + a.writeLong(segSize, 222L); + + assertEquals(a.readLong(segSize - 1), 111L); + assertEquals(a.readLong(segSize), 222L); + } + + @Test + public void testRoundTripAllValues() { + @Cleanup + SegmentedLongArray a = new SegmentedLongArray(1000); + + long[] testValues = {0, 1, -1, Long.MAX_VALUE, Long.MIN_VALUE, + 255L, 256L, 65535L, 65536L, + Integer.MAX_VALUE, Integer.MIN_VALUE}; + + for (int i = 0; i < testValues.length; i++) { + a.writeLong(i, testValues[i]); + } + for (int i = 0; i < testValues.length; i++) { + assertEquals(a.readLong(i), testValues[i]); + } + } + + @Test + public void testCloseReleasesMemory() { + SegmentedLongArray a = new SegmentedLongArray(100); + a.close(); + assertThrows(NullPointerException.class, () -> a.readLong(0)); + } + + @Test + public void testZeroCapacityRejected() { + assertThrows(IllegalArgumentException.class, () -> { + @Cleanup + SegmentedLongArray ignored = new SegmentedLongArray(0); + }); + } + + @Test + public void testNegativeCapacityRejected() { + assertThrows(IllegalArgumentException.class, () -> { + @Cleanup + SegmentedLongArray ignored = new SegmentedLongArray(-1); + }); + } + + @Test + public void testGrowAfterShrink() { + long segSize = TEST_SEGMENT_SIZE; + @Cleanup + SegmentedLongArray a = new SegmentedLongArray(segSize, TEST_SEGMENT_SIZE); + a.increaseCapacity(); + a.increaseCapacity(); + a.shrink(segSize); + assertEquals(a.getCapacity(), segSize); + + a.writeLong(0, 42L); + + a.increaseCapacity(); + a.writeLong(segSize, 99L); + + assertEquals(a.readLong(0), 42L); + assertEquals(a.readLong(segSize), 99L); + } + + @Test + public void testShrinkNoOpWhenEqual() { + @Cleanup + SegmentedLongArray a = new SegmentedLongArray(100); + a.increaseCapacity(); // 200 + a.shrink(200); // newCapacity == capacity, no-op + assertEquals(a.getCapacity(), 200); + } + + @Test + public void testShrinkNoOpWhenExceeds() { + @Cleanup + SegmentedLongArray a = new SegmentedLongArray(100); + a.increaseCapacity(); // 200 + a.shrink(300); // newCapacity > capacity, no-op + assertEquals(a.getCapacity(), 200); + } + + @Test + public void testNegativeOffset() { + @Cleanup + SegmentedLongArray a = new SegmentedLongArray(10); + assertThrows(IndexOutOfBoundsException.class, () -> a.readLong(-1)); + } + + @Test + public void testSmallInitialCapacityDoesNotAllocateFullSegment() { + @Cleanup + SegmentedLongArray a = new SegmentedLongArray(48, TEST_SEGMENT_SIZE); + assertEquals(a.getCapacity(), 48); + assertEquals(a.bytesCapacity(), 48 * 8); + assertTrue(a.bytesCapacity() < TEST_SEGMENT_SIZE * 8); + } + + @Test + public void testPartialLastSegmentMapping() { + long seg = TEST_SEGMENT_SIZE; + long cap = seg + 1000; + @Cleanup + SegmentedLongArray a = new SegmentedLongArray(cap, TEST_SEGMENT_SIZE); + assertEquals(a.getCapacity(), cap); + assertEquals(a.bytesCapacity(), cap * 8); + a.writeLong(0, 1L); + a.writeLong(seg - 1, 2L); + a.writeLong(seg, 3L); + a.writeLong(cap - 1, 4L); + assertEquals(a.readLong(0), 1L); + assertEquals(a.readLong(seg - 1), 2L); + assertEquals(a.readLong(seg), 3L); + assertEquals(a.readLong(cap - 1), 4L); + } + + @Test + public void testIncreaseCapacityPromotesPartialLastToFull() { + long seg = TEST_SEGMENT_SIZE; + @Cleanup + SegmentedLongArray a = new SegmentedLongArray(seg * 2 + seg / 2, TEST_SEGMENT_SIZE); + long partialBase = seg * 2; + a.writeLong(partialBase, 99L); + a.writeLong(partialBase + seg / 2 - 1, 100L); + long capacityBefore = a.getCapacity(); + a.increaseCapacity(); + assertTrue(a.getCapacity() > capacityBefore); + assertEquals(a.readLong(partialBase), 99L); + assertEquals(a.readLong(partialBase + seg / 2 - 1), 100L); + assertEquals(a.bytesCapacity(), a.getCapacity() * 8); + } + + @Test + public void testBytesCapacityTracksPhysicalThroughGrowAndShrink() { + @Cleanup + SegmentedLongArray a = new SegmentedLongArray(1000); + assertEquals(a.getCapacity(), 1000L); + assertEquals(a.bytesCapacity(), 8000L); + for (int i = 0; i < 5; i++) { + a.increaseCapacity(); + assertEquals(a.bytesCapacity(), a.getCapacity() * 8); + } + a.shrink(2000); + assertEquals(a.bytesCapacity(), a.getCapacity() * 8); + } + + @Test + public void testOffsetMappingAndCapacityAcrossOperations() { + long seg = TEST_SEGMENT_SIZE; + @Cleanup + SegmentedLongArray a = new SegmentedLongArray(50, TEST_SEGMENT_SIZE); + assertEquals(a.bytesCapacity(), a.getCapacity() * 8); + + for (int i = 0; i < 5; i++) { + a.increaseCapacity(); + assertEquals(a.bytesCapacity(), a.getCapacity() * 8); + } + a.ensureCapacity(seg * 3 + 1000); + assertEquals(a.bytesCapacity(), a.getCapacity() * 8); + a.writeLong(0, 1L); + a.writeLong(seg - 1, 2L); + a.writeLong(seg, 3L); + a.writeLong(a.getCapacity() - 1, 4L); + assertEquals(a.readLong(0), 1L); + assertEquals(a.readLong(seg - 1), 2L); + assertEquals(a.readLong(seg), 3L); + assertEquals(a.readLong(a.getCapacity() - 1), 4L); + + a.shrink(seg * 2 + 500); + assertEquals(a.bytesCapacity(), a.getCapacity() * 8); + a.shrink(seg); + assertEquals(a.bytesCapacity(), a.getCapacity() * 8); + a.ensureCapacity(seg * 2 + 100); + assertEquals(a.bytesCapacity(), a.getCapacity() * 8); + a.writeLong(seg, 7L); + assertEquals(a.readLong(seg), 7L); + } + + @Test + public void testAllocatedBytesDeltaAtEveryMutationSite() { + long seg = TEST_SEGMENT_SIZE; + @Cleanup + SegmentedLongArray a = new SegmentedLongArray(100, TEST_SEGMENT_SIZE); + assertEquals(a.bytesCapacity(), a.getCapacity() * 8); + + for (int i = 0; i < 4; i++) { + a.increaseCapacity(); + assertEquals(a.bytesCapacity(), a.getCapacity() * 8); + } + a.ensureCapacity(seg * 3 + 1000); + assertEquals(a.bytesCapacity(), a.getCapacity() * 8); + a.shrink(seg + 500); + assertEquals(a.bytesCapacity(), a.getCapacity() * 8); + a.shrink(seg / 2); + assertEquals(a.bytesCapacity(), a.getCapacity() * 8); + a.ensureCapacity(seg * 2 + 100); + assertEquals(a.bytesCapacity(), a.getCapacity() * 8); + } + + @Test + public void testShrinkTrimsOverallocatedContainer() { + long seg = TEST_SEGMENT_SIZE; + @Cleanup + SegmentedLongArray a = new SegmentedLongArray(50, TEST_SEGMENT_SIZE); + a.ensureCapacity(seg * 30); + assertTrue(a.getCapacity() >= seg * 30); + a.shrink(seg / 2); + assertEquals(a.bytesCapacity(), a.getCapacity() * 8); + a.writeLong(0, 1L); + a.writeLong(a.getCapacity() - 1, 2L); + assertEquals(a.readLong(0), 1L); + assertEquals(a.readLong(a.getCapacity() - 1), 2L); + } + + @Test + public void testCustomSegmentSizeRejectsNonPowerOfTwo() { + assertThrows(IllegalArgumentException.class, () -> { + @Cleanup + SegmentedLongArray ignored = new SegmentedLongArray(100, 1000); + }); + } + + @Test + public void testCustomSegmentSizeRejectsZero() { + assertThrows(IllegalArgumentException.class, () -> { + @Cleanup + SegmentedLongArray ignored = new SegmentedLongArray(100, 0); + }); + } + + @Test + public void testCustomSegmentSizeRejectsNegative() { + assertThrows(IllegalArgumentException.class, () -> { + @Cleanup + SegmentedLongArray ignored = new SegmentedLongArray(100, -1024); + }); + } } From 1a26bc2ba6f44ca9358c34be242f97c7fcc5a431 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Fri, 10 Jul 2026 21:07:10 +0800 Subject: [PATCH 199/213] [fix][broker] Remove lock contention in delayed delivery stats read paths (#25990) --- .../InMemoryDelayedDeliveryTracker.java | 10 +- .../bucket/BucketDelayedDeliveryTracker.java | 116 ++++--- .../BucketDelayedMessageIndexStats.java | 2 +- .../delayed/bucket/ImmutableBucket.java | 12 +- ...PersistentDispatcherMultipleConsumers.java | 26 +- ...entDispatcherMultipleConsumersClassic.java | 26 +- .../delayed/MockBucketSnapshotStorage.java | 11 +- .../BucketDelayedDeliveryTrackerTest.java | 293 ++++++++++++++++++ 8 files changed, 420 insertions(+), 76 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java index e94bb32a6f45c..fc4fe56b0f17c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java @@ -24,6 +24,7 @@ import java.util.NavigableSet; import java.util.TreeSet; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicLong; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Position; @@ -52,6 +53,10 @@ public class InMemoryDelayedDeliveryTracker extends AbstractDelayedDeliveryTrack // Track whether we have seen all messages with fixed delay so far. private boolean messagesHaveFixedDelay = true; + // Count of delayed messages in the tracker, maintained incrementally so that stats reads + // do not contend with mutation paths (#24430 / #25990). + private final AtomicLong delayedMessagesCount = new AtomicLong(0); + InMemoryDelayedDeliveryTracker(AbstractPersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis, boolean isDelayedDeliveryDeliverAtTimeStrict, @@ -90,6 +95,7 @@ public boolean addMessage(long ledgerId, long entryId, long deliverAt) { } priorityQueue.add(deliverAt, ledgerId, entryId); + delayedMessagesCount.incrementAndGet(); updateTimer(); checkAndUpdateHighest(deliverAt); @@ -141,6 +147,7 @@ public NavigableSet getScheduledMessages(int maxMessages) { positions.add(PositionFactory.create(ledgerId, entryId)); priorityQueue.pop(); + delayedMessagesCount.decrementAndGet(); --n; } @@ -161,12 +168,13 @@ public NavigableSet getScheduledMessages(int maxMessages) { @Override public CompletableFuture clear() { this.priorityQueue.clear(); + this.delayedMessagesCount.set(0); return CompletableFuture.completedFuture(null); } @Override public long getNumberOfDelayedMessages() { - return priorityQueue.size(); + return delayedMessagesCount.get(); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index 3a889ac223524..df731ee2c38d2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -31,7 +31,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.NavigableSet; @@ -110,6 +109,14 @@ public static record SnapshotKey(long ledgerId, long entryId) {} @VisibleForTesting private final RangeMap immutableBuckets; + @Getter + @VisibleForTesting + private final AtomicLong bucketsCount = new AtomicLong(0); + + @Getter + @VisibleForTesting + private final AtomicLong totalSnapshotLengthBytes = new AtomicLong(0); + private final ConcurrentHashMap snapshotSegmentLastIndexMap; private final BucketDelayedMessageIndexStats stats; @@ -243,15 +250,19 @@ private synchronized long recoverBucketSnapshot() throws RecoverDelayedDeliveryT for (Map.Entry, ImmutableBucket> mapEntry : toBeDeletedBucketMap.entrySet()) { Range key = mapEntry.getKey(); ImmutableBucket immutableBucket = mapEntry.getValue(); - immutableBucketMap.remove(key); + removeBucket(key); // delete asynchronously without waiting for completion immutableBucket.asyncDeleteBucketSnapshot(stats); } MutableLong numberDelayedMessages = new MutableLong(0); - immutableBucketMap.values().forEach(bucket -> { + long totalLength = 0; + for (ImmutableBucket bucket : immutableBucketMap.values()) { numberDelayedMessages.add(bucket.numberBucketDelayedMessages); - }); + totalLength += bucket.getSnapshotLength(); + } + totalSnapshotLengthBytes.set(totalLength); + bucketsCount.set(immutableBuckets.asMapOfRanges().size()); log.info("[{}] Recover delayed message index bucket snapshot finish, buckets: {}, numberDelayedMessages: {}", context.getName(), immutableBucketMap.size(), numberDelayedMessages.longValue()); @@ -288,12 +299,16 @@ private CompletableFuture> handleRecoverBucketSnapshotEntry(I private synchronized void putAndCleanOverlapRange(Range range, ImmutableBucket immutableBucket, Map, ImmutableBucket> toBeDeletedBucketMap) { - RangeMap subRangeMap = immutableBuckets.subRangeMap(range); + Map, ImmutableBucket> subRangeMap = immutableBuckets.subRangeMap(range).asMapOfRanges(); boolean canPut = false; - if (!subRangeMap.asMapOfRanges().isEmpty()) { - for (Map.Entry, ImmutableBucket> rangeEntry : subRangeMap.asMapOfRanges().entrySet()) { - if (range.encloses(rangeEntry.getKey())) { - toBeDeletedBucketMap.put(rangeEntry.getKey(), rangeEntry.getValue()); + if (!subRangeMap.isEmpty()) { + for (Map.Entry, ImmutableBucket> rangeEntry : subRangeMap.entrySet()) { + // Use original key instead of truncated key for encloses check + ImmutableBucket bucket = rangeEntry.getValue(); + Range originalKey = Range.closed(bucket.startLedgerId, bucket.endLedgerId); + + if (range.encloses(originalKey)) { + toBeDeletedBucketMap.put(originalKey, bucket); canPut = true; } } @@ -302,7 +317,7 @@ private synchronized void putAndCleanOverlapRange(Range range, ImmutableBu } if (canPut) { - immutableBuckets.put(range, immutableBucket); + putBucket(range, immutableBucket); } } @@ -329,7 +344,7 @@ private void afterCreateImmutableBucket(Pair immu long startTime) { if (immutableBucketDelayedIndexPair != null) { ImmutableBucket immutableBucket = immutableBucketDelayedIndexPair.getLeft(); - immutableBuckets.put(Range.closed(immutableBucket.startLedgerId, immutableBucket.endLedgerId), + putBucket(Range.closed(immutableBucket.startLedgerId, immutableBucket.endLedgerId), immutableBucket); DelayedIndex lastDelayedIndex = immutableBucketDelayedIndexPair.getRight(); @@ -341,7 +356,12 @@ private void afterCreateImmutableBucket(Pair immu CompletableFuture future = createFuture.handle((bucketId, ex) -> { if (ex == null) { immutableBucket.setSnapshotSegments(null); - immutableBucket.asyncUpdateSnapshotLength(); + immutableBucket.asyncUpdateSnapshotLength() + .thenAccept(newLength -> { + synchronized (BucketDelayedDeliveryTracker.this) { + updateBucketSnapshotLength(immutableBucket, newLength); + } + }); log.info("[{}] Create bucket snapshot finish, bucketKey: {}", context.getName(), immutableBucket.bucketKey()); @@ -368,7 +388,7 @@ private void afterCreateImmutableBucket(Pair immu }); immutableBucket.setCurrentSegmentEntryId(immutableBucket.lastSegmentEntryId); - immutableBuckets.asMapOfRanges().remove( + removeBucket( Range.closed(immutableBucket.startLedgerId, immutableBucket.endLedgerId)); snapshotSegmentLastIndexMap.remove( new SnapshotKey(lastDelayedIndex.getLedgerId(), lastDelayedIndex.getEntryId())); @@ -406,7 +426,7 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver afterCreateImmutableBucket(immutableBucketDelayedIndexPair, createStartTime); lastMutableBucket.resetLastMutableBucketRange(); - if (maxNumBuckets > 0 && immutableBuckets.asMapOfRanges().size() > maxNumBuckets + if (maxNumBuckets > 0 && bucketsCount.get() > maxNumBuckets && (trimFuture == null || trimFuture.isDone())) { trimFuture = asyncTrimImmutableBuckets() .thenCompose(ignore -> asyncMergeBucketSnapshot()) @@ -484,11 +504,10 @@ synchronized List selectMergedBuckets(final List asyncMergeBucketSnapshot() { - List immutableBucketList = immutableBuckets.asMapOfRanges().values().stream().toList(); - if (maxNumBuckets <= 0 || immutableBucketList.size() <= maxNumBuckets) { + if (maxNumBuckets <= 0 || bucketsCount.get() <= maxNumBuckets) { return CompletableFuture.completedFuture(null); } - + List immutableBucketList = immutableBuckets.asMapOfRanges().values().stream().toList(); List toBeMergeImmutableBuckets = selectMergedBuckets(immutableBucketList, MAX_MERGE_NUM); if (toBeMergeImmutableBuckets.isEmpty()) { @@ -521,7 +540,7 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot() { stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.merge); } else { log.info("[{}] Merge bucket snapshot finish, bucketKeys: {}, bucketNum: {}", - context.getName(), bucketsStr, immutableBuckets.asMapOfRanges().size()); + context.getName(), bucketsStr, bucketsCount.get()); stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.merge, System.currentTimeMillis() - mergeStartTime); @@ -588,8 +607,7 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(List getScheduledMessages(int maxMessages) synchronized (BucketDelayedDeliveryTracker.this) { this.snapshotSegmentLastIndexMap.remove(snapshotKey); if (CollectionUtils.isEmpty(indexList)) { - immutableBuckets.asMapOfRanges() - .remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId)); + removeBucket(Range.closed(bucket.startLedgerId, bucket.endLedgerId)); bucket.asyncDeleteBucketSnapshot(stats); return; } @@ -817,14 +834,16 @@ public CompletableFuture closeAsync() { } private CompletableFuture cleanImmutableBuckets() { + Map, ImmutableBucket> bucketsToDelete = + new HashMap<>(immutableBuckets.asMapOfRanges()); + List> futures = new ArrayList<>(); - Iterator iterator = immutableBuckets.asMapOfRanges().values().iterator(); - while (iterator.hasNext()) { - ImmutableBucket bucket = iterator.next(); - futures.add(bucket.clear(stats)); + bucketsToDelete.forEach((range, bucket) -> { + removeBucket(range); numberDelayedMessages.addAndGet(-bucket.getNumberBucketDelayedMessages()); - iterator.remove(); - } + futures.add(bucket.clear(stats)); + }); + return FutureUtil.waitForAll(futures); } @@ -848,13 +867,9 @@ public synchronized boolean containsMessage(long ledgerId, long entryId) { public Map genTopicMetricMap() { - stats.recordNumOfBuckets(immutableBuckets.asMapOfRanges().size() + 1); + stats.recordNumOfBuckets((int) (bucketsCount.get() + 1)); stats.recordDelayedMessageIndexLoaded(this.sharedBucketPriorityQueue.size() + this.lastMutableBucket.size()); - MutableLong totalSnapshotLength = new MutableLong(); - immutableBuckets.asMapOfRanges().values().forEach(immutableBucket -> { - totalSnapshotLength.add(immutableBucket.getSnapshotLength()); - }); - stats.recordBucketSnapshotSizeBytes(totalSnapshotLength.longValue()); + stats.recordBucketSnapshotSizeBytes(totalSnapshotLengthBytes.get()); return stats.genTopicMetricMap(); } @@ -903,7 +918,7 @@ private CompletableFuture deleteBucketSnapshot(String ledgerName, } synchronized (this) { snapshotSegmentLastIndexMap.entrySet().removeIf(entry -> entry.getValue() == bucket); - immutableBuckets.remove(range); + removeBucket(range); numberDelayedMessages.addAndGet(-bucket.getNumberBucketDelayedMessages()); } return null; @@ -915,4 +930,35 @@ private Long firstActiveLedgerId() { Position mdp = cursor.getMarkDeletedPosition(); return mdp == null ? null : mdp.getLedgerId(); } + + private void putBucket(Range range, ImmutableBucket bucket) { + long removedLength = immutableBuckets.subRangeMap(range).asMapOfRanges().values().stream() + .mapToLong(ImmutableBucket::getSnapshotLength) + .sum(); + + immutableBuckets.put(range, bucket); + bucketsCount.set(immutableBuckets.asMapOfRanges().size()); + totalSnapshotLengthBytes.addAndGet(bucket.getSnapshotLength() - removedLength); + } + + private void removeBucket(Range range) { + // Use exact key matching - all callers should provide exact keys + ImmutableBucket bucket = immutableBuckets.asMapOfRanges().get(range); + + if (bucket != null) { + // Remove even if snapshot length is 0 (for newly created buckets) + immutableBuckets.asMapOfRanges().remove(range); + bucketsCount.set(immutableBuckets.asMapOfRanges().size()); + totalSnapshotLengthBytes.addAndGet(-bucket.getSnapshotLength()); + } + } + + private void updateBucketSnapshotLength(ImmutableBucket bucket, long newLength) { + if (!immutableBuckets.asMapOfRanges().containsValue(bucket)) { + return; + } + long oldLength = bucket.getSnapshotLength(); + bucket.setSnapshotLength(newLength); + totalSnapshotLengthBytes.addAndGet(newLength - oldLength); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexStats.java index b9e6c7dc64c7c..502ca9f3d4edd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexStats.java @@ -59,7 +59,7 @@ enum Type { public BucketDelayedMessageIndexStats() { } - public Map genTopicMetricMap() { + public synchronized Map genTopicMetricMap() { Map metrics = new HashMap<>(); metrics.put(BUCKET_TOTAL_NAME, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java index 7d7b3a2face64..57f48034cbeb4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java @@ -121,9 +121,9 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b bucketId, nextSegmentEntryId, ex); } }), BucketSnapshotPersistenceException.class, MaxRetryTimes) - .thenApply(bucketSnapshotSegments -> { + .thenCompose(bucketSnapshotSegments -> { if (CollectionUtils.isEmpty(bucketSnapshotSegments)) { - return Collections.emptyList(); + return CompletableFuture.completedFuture(Collections.emptyList()); } SnapshotSegment snapshotSegment = @@ -131,9 +131,11 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b List indexList = snapshotSegment.getIndexesList(); this.setCurrentSegmentEntryId(nextSegmentEntryId); if (isRecover) { - this.asyncUpdateSnapshotLength(); + return this.asyncUpdateSnapshotLength() + .thenAccept(this::setSnapshotLength) + .thenApply(__ -> indexList); } - return indexList; + return CompletableFuture.completedFuture(indexList); }); }); } @@ -223,8 +225,6 @@ protected CompletableFuture asyncUpdateSnapshotLength() { if (ex != null) { log.error("[{}] Failed to get snapshot length, bucketId: {}, bucketKey: {}", dispatcherName, bucketId, bucketKey(), ex); - } else { - setSnapshotLength(length); } }); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index b85f3508d3dea..04d362a93f183 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -91,7 +91,7 @@ public class PersistentDispatcherMultipleConsumers extends AbstractPersistentDis protected final MessageRedeliveryController redeliveryMessages; protected final RedeliveryTracker redeliveryTracker; - private Optional delayedDeliveryTracker = Optional.empty(); + private volatile Optional delayedDeliveryTracker = Optional.empty(); protected volatile boolean havePendingRead = false; protected volatile boolean havePendingReplayRead = false; @@ -1363,13 +1363,12 @@ protected boolean isNormalReadAllowed() { } - - protected synchronized boolean shouldPauseDeliveryForDelayTracker() { - return delayedDeliveryTracker.isPresent() && delayedDeliveryTracker.get().shouldPauseAllDeliveries(); + protected boolean shouldPauseDeliveryForDelayTracker() { + return delayedDeliveryTracker.map(DelayedDeliveryTracker::shouldPauseAllDeliveries).orElse(false); } @Override - public synchronized long getNumberOfDelayedMessages() { + public long getNumberOfDelayedMessages() { return delayedDeliveryTracker.map(DelayedDeliveryTracker::getNumberOfDelayedMessages).orElse(0L); } @@ -1453,20 +1452,15 @@ public PersistentTopic getTopic() { } - public synchronized long getDelayedTrackerMemoryUsage() { + public long getDelayedTrackerMemoryUsage() { return delayedDeliveryTracker.map(DelayedDeliveryTracker::getBufferMemoryUsage).orElse(0L); } - public synchronized Map getBucketDelayedIndexStats() { - if (delayedDeliveryTracker.isEmpty()) { - return Collections.emptyMap(); - } - - if (delayedDeliveryTracker.get() instanceof BucketDelayedDeliveryTracker) { - return ((BucketDelayedDeliveryTracker) delayedDeliveryTracker.get()).genTopicMetricMap(); - } - - return Collections.emptyMap(); + public Map getBucketDelayedIndexStats() { + return delayedDeliveryTracker + .filter(BucketDelayedDeliveryTracker.class::isInstance) + .map(tracker -> ((BucketDelayedDeliveryTracker) tracker).genTopicMetricMap()) + .orElse(Collections.emptyMap()); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java index 79fa5c157af71..5f62f5a2823e5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumersClassic.java @@ -92,7 +92,7 @@ public class PersistentDispatcherMultipleConsumersClassic extends AbstractPersis protected final MessageRedeliveryController redeliveryMessages; protected final RedeliveryTracker redeliveryTracker; - private Optional delayedDeliveryTracker = Optional.empty(); + private volatile Optional delayedDeliveryTracker = Optional.empty(); protected volatile boolean havePendingRead = false; protected volatile boolean havePendingReplayRead = false; @@ -1200,12 +1200,12 @@ protected boolean hasConsumersNeededNormalRead() { return true; } - protected synchronized boolean shouldPauseDeliveryForDelayTracker() { - return delayedDeliveryTracker.isPresent() && delayedDeliveryTracker.get().shouldPauseAllDeliveries(); + protected boolean shouldPauseDeliveryForDelayTracker() { + return delayedDeliveryTracker.map(DelayedDeliveryTracker::shouldPauseAllDeliveries).orElse(false); } @Override - public synchronized long getNumberOfDelayedMessages() { + public long getNumberOfDelayedMessages() { return delayedDeliveryTracker.map(DelayedDeliveryTracker::getNumberOfDelayedMessages).orElse(0L); } @@ -1283,21 +1283,15 @@ public PersistentTopic getTopic() { return topic; } - - public synchronized long getDelayedTrackerMemoryUsage() { + public long getDelayedTrackerMemoryUsage() { return delayedDeliveryTracker.map(DelayedDeliveryTracker::getBufferMemoryUsage).orElse(0L); } - public synchronized Map getBucketDelayedIndexStats() { - if (delayedDeliveryTracker.isEmpty()) { - return Collections.emptyMap(); - } - - if (delayedDeliveryTracker.get() instanceof BucketDelayedDeliveryTracker) { - return ((BucketDelayedDeliveryTracker) delayedDeliveryTracker.get()).genTopicMetricMap(); - } - - return Collections.emptyMap(); + public Map getBucketDelayedIndexStats() { + return delayedDeliveryTracker + .filter(BucketDelayedDeliveryTracker.class::isInstance) + .map(tracker -> ((BucketDelayedDeliveryTracker) tracker).genTopicMetricMap()) + .orElse(Collections.emptyMap()); } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java index a1f5b554b1520..67776c236d81d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java @@ -193,6 +193,15 @@ public void clean() { } } bucketSnapshots.clear(); - executorService.shutdownNow(); + // Gracefully shutdown: allow pending tasks to complete + executorService.shutdown(); + try { + if (!executorService.awaitTermination(10, TimeUnit.SECONDS)) { + executorService.shutdownNow(); + } + } catch (InterruptedException e) { + executorService.shutdownNow(); + Thread.currentThread().interrupt(); + } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java index 231e3690a6038..f26204991d955 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java @@ -22,6 +22,7 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotSame; @@ -911,4 +912,296 @@ public void testGetScheduledMessagesWhenAllOrphaned() throws Exception { ts.close(); } + + /** + * Test that overlapping buckets are correctly cleaned up during recovery. + * This verifies the fix for the subRangeMap clipped key issue where + * putAndCleanOverlapRange would store clipped keys that couldn't be + * removed by exact key matching in removeBucket(). + */ + @Test + public void testOverlappingBucketsCleanupDuringRecovery() throws Exception { + // Setup mocks + AbstractPersistentDispatcherMultipleConsumers testDispatcher = + mock(AbstractPersistentDispatcherMultipleConsumers.class); + Clock testClock = mock(Clock.class); + AtomicLong testClockTime = new AtomicLong(); + when(testClock.millis()).then(x -> testClockTime.get()); + + MockBucketSnapshotStorage storage = new MockBucketSnapshotStorage(); + storage.start(); + + ManagedCursor cursor = new MockManagedCursor("test_overlap_cursor"); + doReturn(cursor).when(testDispatcher).getCursor(); + doReturn("persistent://public/default/testOverlap / " + cursor.getName()) + .when(testDispatcher).getName(); + + try { + // Create first tracker with small minIndexCountPerBucket + BucketDelayedDeliveryTracker tracker1 = new BucketDelayedDeliveryTracker( + testDispatcher, timer, 100000, testClock, true, storage, + 3, TimeUnit.MILLISECONDS.toMillis(10), -1, 50); + + // Add messages to create multiple immutable buckets + for (int i = 1; i <= 12; i++) { + tracker1.addMessage(i, i, i * 10); + } + + // Wait for all bucket operations to complete + Awaitility.await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> { + assertTrue(tracker1.getImmutableBuckets().asMapOfRanges().values().stream() + .noneMatch(x -> x.merging), + "All buckets should finish merging"); + assertTrue(tracker1.getImmutableBuckets().asMapOfRanges().size() >= 2, + "Should have created multiple buckets"); + }); + + int bucketCountBeforeClose = tracker1.getImmutableBuckets().asMapOfRanges().size(); + + tracker1.close(); + + // Create second tracker - triggers recovery with putAndCleanOverlapRange + BucketDelayedDeliveryTracker tracker2 = new BucketDelayedDeliveryTracker( + testDispatcher, timer, 100000, testClock, true, storage, + 3, TimeUnit.MILLISECONDS.toMillis(10), -1, 50); + + // Verify buckets were recovered + int bucketCountAfterRecovery = tracker2.getImmutableBuckets().asMapOfRanges().size(); + assertTrue(bucketCountAfterRecovery > 0, "Should have recovered buckets"); + + // Key assertion: verify no orphaned buckets remain + // If clipped keys weren't fixed, removeBucket() would fail and buckets would accumulate + assertTrue(bucketCountAfterRecovery <= bucketCountBeforeClose, + String.format("Orphaned buckets detected: %d after recovery > %d before close", + bucketCountAfterRecovery, bucketCountBeforeClose)); + + // Verify messages were recovered + assertTrue(tracker2.getNumberOfDelayedMessages() > 0, + "Should have recovered messages"); + + // Verify snapshot length tracking is correct + long totalSnapshotLength = tracker2.getImmutableBuckets().asMapOfRanges().values().stream() + .mapToLong(ImmutableBucket::getSnapshotLength) + .sum(); + assertTrue(totalSnapshotLength >= 0, + "Snapshot length tracking broken - likely due to failed removeBucket()"); + + tracker2.close(); + } finally { + storage.clean(); + } + } + + /** + * Test that putAndCleanOverlapRange correctly uses original keys instead of truncated keys + * when checking if a new range encloses existing buckets. + * + * This prevents the bug where a truncated key from subRangeMap() would incorrectly pass + * the encloses() check, causing a bucket to be replaced when it shouldn't be. + */ + @Test + public void testPutAndCleanOverlapRangeWithTruncatedKeys() throws Exception { + // Setup mocks + AbstractPersistentDispatcherMultipleConsumers testDispatcher = + mock(AbstractPersistentDispatcherMultipleConsumers.class); + Clock testClock = mock(Clock.class); + AtomicLong testClockTime = new AtomicLong(); + when(testClock.millis()).then(x -> testClockTime.get()); + + MockBucketSnapshotStorage storage = new MockBucketSnapshotStorage(); + storage.start(); + + ManagedCursor cursor = new MockManagedCursor("test_truncated_cursor"); + doReturn(cursor).when(testDispatcher).getCursor(); + doReturn("persistent://public/default/testTruncated / " + cursor.getName()) + .when(testDispatcher).getName(); + + try { + // Create tracker + BucketDelayedDeliveryTracker tracker = new BucketDelayedDeliveryTracker( + testDispatcher, timer, 100000, testClock, true, storage, + 3, TimeUnit.MILLISECONDS.toMillis(10), -1, 50); + + // Add messages to create a bucket [1-6] + for (int i = 1; i <= 6; i++) { + tracker.addMessage(i, i, i * 10); + } + + // Wait for bucket to be created + Awaitility.await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> { + assertTrue(tracker.getImmutableBuckets().asMapOfRanges().size() >= 1, + "Should have created at least one bucket"); + }); + + int initialBucketCount = tracker.getImmutableBuckets().asMapOfRanges().size(); + long initialMessageCount = tracker.getNumberOfDelayedMessages(); + + // Now add messages that would create a bucket [7-9] + // This should NOT replace the existing bucket [1-6] + for (int i = 7; i <= 9; i++) { + tracker.addMessage(i, i, i * 10); + } + + // Wait for new bucket operations + Awaitility.await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> { + assertTrue(tracker.getImmutableBuckets().asMapOfRanges().values().stream() + .noneMatch(x -> x.merging), + "All buckets should finish processing"); + }); + + // Verify bucket count increased (or stayed same if they got merged) + int finalBucketCount = tracker.getImmutableBuckets().asMapOfRanges().size(); + assertTrue(finalBucketCount >= initialBucketCount, + "Bucket count should not decrease when adding non-overlapping ranges"); + + // Verify all messages are tracked + long finalMessageCount = tracker.getNumberOfDelayedMessages(); + assertTrue(finalMessageCount >= initialMessageCount, + String.format("Message count should not decrease: initial=%d, final=%d", + initialMessageCount, finalMessageCount)); + + // Verify no bucket was incorrectly replaced + // If putAndCleanOverlapRange used truncated keys, it might have incorrectly + // removed a bucket that shouldn't have been removed + tracker.getImmutableBuckets().asMapOfRanges().forEach((range, bucket) -> { + assertTrue(bucket.getNumberBucketDelayedMessages() > 0, + "All buckets should have messages - bucket " + range + " is empty"); + }); + + tracker.close(); + } finally { + storage.clean(); + } + } + + @Test + public void testLateSnapshotLengthUpdateAfterClearDoesNotInflateCounter() throws Exception { + AbstractPersistentDispatcherMultipleConsumers testDispatcher = + mock(AbstractPersistentDispatcherMultipleConsumers.class); + Clock testClock = mock(Clock.class); + AtomicLong testClockTime = new AtomicLong(); + when(testClock.millis()).then(x -> testClockTime.get()); + + MockBucketSnapshotStorage storage = new MockBucketSnapshotStorage(); + storage.start(); + MockBucketSnapshotStorage spyStorage = spy(storage); + + CompletableFuture blockedLength = new CompletableFuture<>(); + when(spyStorage.getBucketSnapshotLength(anyLong())).thenReturn(blockedLength); + + ManagedCursor cursor = new MockManagedCursor("test_late_update_cursor"); + doReturn(cursor).when(testDispatcher).getCursor(); + doReturn("persistent://public/default/testLateUpdate / " + cursor.getName()) + .when(testDispatcher).getName(); + + try { + BucketDelayedDeliveryTracker tracker = new BucketDelayedDeliveryTracker( + testDispatcher, timer, 100000, testClock, true, spyStorage, + 3, TimeUnit.MILLISECONDS.toMillis(10), -1, 50); + + for (int i = 1; i <= 6; i++) { + tracker.addMessage(i, i, i * 10); + } + + Awaitility.await().untilAsserted(() -> + assertTrue(tracker.getBucketsCount().get() >= 1, + "Should have created at least one immutable bucket")); + assertCountersConsistent(tracker); + + tracker.clear(); + + assertEquals(tracker.getBucketsCount().get(), 0, "All buckets should be removed"); + assertCountersConsistent(tracker); + + blockedLength.complete(999_999L); + + Awaitility.await().untilAsserted(() -> { + assertEquals(tracker.getTotalSnapshotLengthBytes().get(), 0, + "Late length update inflated totalSnapshotLengthBytes after clear"); + }); + + tracker.close(); + } finally { + storage.clean(); + } + } + + @Test + public void testLateSnapshotLengthUpdateAfterTrimDoesNotInflateCounter() throws Exception { + AbstractPersistentDispatcherMultipleConsumers testDispatcher = + mock(AbstractPersistentDispatcherMultipleConsumers.class); + Clock testClock = mock(Clock.class); + AtomicLong testClockTime = new AtomicLong(); + when(testClock.millis()).then(x -> testClockTime.get()); + + MockBucketSnapshotStorage storage = new MockBucketSnapshotStorage(); + storage.start(); + MockBucketSnapshotStorage spyStorage = spy(storage); + + CompletableFuture blockedLength = new CompletableFuture<>(); + when(spyStorage.getBucketSnapshotLength(anyLong())).thenReturn(blockedLength); + + ManagedCursor spyCursor = spy(new MockManagedCursor("test_late_trim_cursor")); + AtomicLong markDeletedLedger = new AtomicLong(0); + when(spyCursor.getMarkDeletedPosition()).thenAnswer(inv -> + PositionFactory.create(markDeletedLedger.get(), 0)); + ManagedLedger mockLedger = mock(ManagedLedger.class); + when(mockLedger.getName()).thenReturn("test_ledger"); + when(spyCursor.getManagedLedger()).thenReturn(mockLedger); + + doReturn(spyCursor).when(testDispatcher).getCursor(); + doReturn("persistent://public/default/testLateTrim / " + spyCursor.getName()) + .when(testDispatcher).getName(); + + try { + BucketDelayedDeliveryTracker tracker = new BucketDelayedDeliveryTracker( + testDispatcher, timer, 100000, testClock, true, spyStorage, + 3, TimeUnit.MILLISECONDS.toMillis(10), -1, 3); + + for (int i = 1; i <= 12; i++) { + tracker.addMessage(i, i, i * 10); + } + + Awaitility.await().untilAsserted(() -> + assertTrue(tracker.getBucketsCount().get() >= 3, + "Should have created at least 3 immutable buckets")); + assertCountersConsistent(tracker); + + markDeletedLedger.set(5); + + for (int i = 13; i <= 15; i++) { + tracker.addMessage(i, i, i * 10); + } + + Awaitility.await().untilAsserted(() -> { + boolean hasOldBucket = tracker.getImmutableBuckets().asMapOfRanges().keySet().stream() + .anyMatch(r -> r.upperEndpoint() < 5); + Assert.assertFalse(hasOldBucket, "Buckets with endLedgerId < 5 should be trimmed"); + }); + assertCountersConsistent(tracker); + + blockedLength.complete(999_999L); + + Awaitility.await().untilAsserted(() -> + assertCountersConsistent(tracker)); + + tracker.close(); + } finally { + storage.clean(); + } + } + + private static void assertCountersConsistent(BucketDelayedDeliveryTracker tracker) { + int liveBucketCount = tracker.getImmutableBuckets().asMapOfRanges().size(); + long liveSnapshotLength = tracker.getImmutableBuckets().asMapOfRanges().values().stream() + .mapToLong(ImmutableBucket::getSnapshotLength) + .sum(); + + assertEquals(tracker.getBucketsCount().get(), liveBucketCount, + String.format("bucketsCount drift: cached=%d live=%d", + tracker.getBucketsCount().get(), liveBucketCount)); + assertEquals(tracker.getTotalSnapshotLengthBytes().get(), liveSnapshotLength, + String.format("totalSnapshotLengthBytes drift: cached=%d live=%d", + tracker.getTotalSnapshotLengthBytes().get(), liveSnapshotLength)); + } } From 34dc3b455dfec968523d02cf18e9c31c7930c7da Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Mon, 20 Jul 2026 14:42:48 +0800 Subject: [PATCH 200/213] [improve][broker] Optimize bucket delayed-delivery bitmap point operations (#26205) --- .../pulsar/broker/delayed/bucket/Bucket.java | 14 ++++++------- .../broker/delayed/bucket/MutableBucket.java | 2 +- .../collections/ConcurrentRoaringBitmap.java | 20 ++++++++++++------- .../common/util/collections/LongBitmap.java | 6 ++++-- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java index d27ca5782449d..2d4ad593cd12b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java @@ -82,20 +82,20 @@ boolean containsMessage(long ledgerId, long entryId) { if (bitSet == null) { return false; } - return bitSet.contains(entryId, entryId + 1); + return bitSet.contains(entryId); } void putIndexBit(long ledgerId, long entryId) { - delayedIndexBitMap.computeIfAbsent(ledgerId, k -> LongBitmaps.create()).add(entryId, entryId + 1); + delayedIndexBitMap.computeIfAbsent(ledgerId, k -> LongBitmaps.create()).add(entryId); } boolean removeIndexBit(long ledgerId, long entryId) { - boolean contained = false; LongBitmap bitSet = delayedIndexBitMap.get(ledgerId); - if (bitSet != null && bitSet.contains(entryId, entryId + 1)) { - contained = true; - bitSet.remove(entryId, entryId + 1); - + if (bitSet == null) { + return false; + } + boolean contained = bitSet.remove(entryId); + if (contained) { if (bitSet.isEmpty()) { delayedIndexBitMap.remove(ledgerId); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java index ffdb652a3820d..02f22f5b17704 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java @@ -106,7 +106,7 @@ Pair createImmutableBucketAndAsyncPersistent( sharedQueue.add(timestamp, ledgerId, entryId); } - bitMap.computeIfAbsent(ledgerId, k -> LongBitmaps.create()).add(entryId, entryId + 1); + bitMap.computeIfAbsent(ledgerId, k -> LongBitmaps.create()).add(entryId); numMessages++; diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java index 774227ed2b351..96ed33c2f8047 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentRoaringBitmap.java @@ -116,32 +116,38 @@ public void add(long from, long to) { } @Override - public void remove(long value) { + public boolean remove(long value) { validateRange(value); long stamp = lock.writeLock(); try { - if (bitmap.checkedRemove((int) value)) { + boolean removed = bitmap.checkedRemove((int) value); + if (removed) { removesSinceTrim++; maybeTrim(); } + return removed; } finally { lock.unlockWrite(stamp); } } @Override - public void remove(long from, long to) { + public boolean remove(long from, long to) { if (to <= from) { - return; + return false; } validateRange(from); validateRange(to - 1); long stamp = lock.writeLock(); try { + long cardinalityBefore = bitmap.getLongCardinality(); bitmap.remove(from, to); - // Range size upper-bounds removals; clamp so a huge range can't overflow the counter. - removesSinceTrim = Math.min(removesSinceTrim + (to - from), TRIM_AFTER_REMOVES); - maybeTrim(); + long removedCount = cardinalityBefore - bitmap.getLongCardinality(); + if (removedCount > 0) { + removesSinceTrim += removedCount; + maybeTrim(); + } + return removedCount > 0; } finally { lock.unlockWrite(stamp); } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongBitmap.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongBitmap.java index 53432eae603dc..e08f05f1bc596 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongBitmap.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/LongBitmap.java @@ -83,8 +83,9 @@ public interface LongBitmap { * * @param value value to remove * @throws IllegalArgumentException if value is outside the supported range + * @return {@code true} if removed, otherwise {@code false} */ - void remove(long value); + boolean remove(long value); /** * Removes all values in the half-open range {@code [from, to)}. @@ -94,8 +95,9 @@ public interface LongBitmap { * @param from inclusive lower bound * @param to exclusive upper bound * @throws IllegalArgumentException if the range exceeds the supported value range + * @return {@code true} if removed, otherwise {@code false} */ - void remove(long from, long to); + boolean remove(long from, long to); /** * Returns whether the bitmap contains the given value. From 7426970bb7079afb1a11fa4a773af1c8f1f55d7a Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Tue, 4 Aug 2026 21:35:28 +0800 Subject: [PATCH 201/213] [fix][broker] Fix bucket delayed delivery counter and dedup consistency (#26251) --- .../pulsar/broker/delayed/bucket/Bucket.java | 176 ---------------- .../broker/delayed/bucket/BucketContext.java | 29 +++ .../bucket/BucketDelayedDeliveryTracker.java | 137 ++++++------- .../bucket/BucketDelayedMessageIndex.java | 83 ++++++++ .../delayed/bucket/ImmutableBucket.java | 191 ++++++++++++++---- .../broker/delayed/bucket/MutableBucket.java | 31 ++- .../BucketDelayedDeliveryTrackerTest.java | 52 ++++- .../bucket/BucketDelayedMessageIndexTest.java | 179 ++++++++++++++++ 8 files changed, 560 insertions(+), 318 deletions(-) delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketContext.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndex.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java deleted file mode 100644 index 2d4ad593cd12b..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.delayed.bucket; - -import static org.apache.bookkeeper.mledger.util.Futures.executeWithRetry; -import static org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker.DELAYED_BUCKET_KEY_PREFIX; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.mledger.ManagedCursor; -import org.apache.bookkeeper.mledger.ManagedLedgerException; -import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; -import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; -import org.apache.pulsar.common.util.Codec; -import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.common.util.collections.LongBitmap; -import org.apache.pulsar.common.util.collections.LongBitmaps; - -@Slf4j -@Data -@AllArgsConstructor -abstract class Bucket { - - static final String DELIMITER = "_"; - static final int MaxRetryTimes = 3; - - protected final String dispatcherName; - - protected final ManagedCursor cursor; - - protected final FutureUtil.Sequencer sequencer; - - protected final BucketSnapshotStorage bucketSnapshotStorage; - - long startLedgerId; - long endLedgerId; - - Map delayedIndexBitMap; - - long numberBucketDelayedMessages; - - int lastSegmentEntryId; - - volatile int currentSegmentEntryId; - - volatile long snapshotLength; - - private volatile Long bucketId; - - private volatile CompletableFuture snapshotCreateFuture; - - - Bucket(String dispatcherName, ManagedCursor cursor, FutureUtil.Sequencer sequencer, - BucketSnapshotStorage storage, long startLedgerId, long endLedgerId) { - this(dispatcherName, cursor, sequencer, storage, startLedgerId, endLedgerId, new HashMap<>(), -1, -1, 0, 0, - null, null); - } - - boolean containsMessage(long ledgerId, long entryId) { - LongBitmap bitSet = delayedIndexBitMap.get(ledgerId); - if (bitSet == null) { - return false; - } - return bitSet.contains(entryId); - } - - void putIndexBit(long ledgerId, long entryId) { - delayedIndexBitMap.computeIfAbsent(ledgerId, k -> LongBitmaps.create()).add(entryId); - } - - boolean removeIndexBit(long ledgerId, long entryId) { - LongBitmap bitSet = delayedIndexBitMap.get(ledgerId); - if (bitSet == null) { - return false; - } - boolean contained = bitSet.remove(entryId); - if (contained) { - if (bitSet.isEmpty()) { - delayedIndexBitMap.remove(ledgerId); - } - - if (numberBucketDelayedMessages > 0) { - numberBucketDelayedMessages--; - } - } - return contained; - } - - String bucketKey() { - return String.join(DELIMITER, DELAYED_BUCKET_KEY_PREFIX, String.valueOf(startLedgerId), - String.valueOf(endLedgerId)); - } - - Optional> getSnapshotCreateFuture() { - return Optional.ofNullable(snapshotCreateFuture); - } - - Optional getBucketId() { - return Optional.ofNullable(bucketId); - } - - long getAndUpdateBucketId() { - Optional bucketIdOptional = getBucketId(); - if (bucketIdOptional.isPresent()) { - return bucketIdOptional.get(); - } - - String bucketIdStr = cursor.getCursorProperties().get(bucketKey()); - long bucketId = Long.parseLong(bucketIdStr); - setBucketId(bucketId); - return bucketId; - } - - CompletableFuture asyncSaveBucketSnapshot( - ImmutableBucket bucket, SnapshotMetadata snapshotMetadata, - List bucketSnapshotSegments) { - final String bucketKey = bucket.bucketKey(); - final String cursorName = Codec.decode(cursor.getName()); - final String topicName = dispatcherName.substring(0, dispatcherName.lastIndexOf(" / " + cursorName)); - return executeWithRetry( - () -> bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, bucketSnapshotSegments, bucketKey, - topicName, cursorName) - .whenComplete((__, ex) -> { - if (ex != null) { - log.warn("[{}] Failed to create bucket snapshot, bucketKey: {}", - dispatcherName, bucketKey, ex); - } - }), BucketSnapshotPersistenceException.class, MaxRetryTimes).thenCompose(newBucketId -> { - bucket.setBucketId(newBucketId); - - return putBucketKeyId(bucketKey, newBucketId).exceptionally(ex -> { - log.warn("[{}] Failed to record bucketId to cursor property, bucketKey: {}, bucketId: {}", - dispatcherName, bucketKey, newBucketId, ex); - return null; - }).thenApply(__ -> newBucketId); - }); - } - - private CompletableFuture putBucketKeyId(String bucketKey, Long bucketId) { - if (bucketId == null) { - return FutureUtil.failedFuture(new NullPointerException("Expected bucketId should not be null")); - } - return sequencer.sequential(() -> { - return executeWithRetry(() -> cursor.putCursorProperty(bucketKey, String.valueOf(bucketId)), - ManagedLedgerException.BadVersionException.class, MaxRetryTimes); - }); - } - - protected CompletableFuture removeBucketCursorProperty(String bucketKey) { - return sequencer.sequential(() -> { - return executeWithRetry(() -> cursor.removeCursorProperty(bucketKey), - ManagedLedgerException.BadVersionException.class, MaxRetryTimes); - }); - } -} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketContext.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketContext.java new file mode 100644 index 0000000000000..cf1d7bfe981cb --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketContext.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.delayed.bucket; + +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.pulsar.common.util.FutureUtil; + +record BucketContext( + String dispatcherName, + ManagedCursor cursor, + FutureUtil.Sequencer sequencer, + BucketSnapshotStorage bucketSnapshotStorage) { +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index df731ee2c38d2..cd009cc63e3b1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -20,13 +20,15 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.apache.bookkeeper.mledger.ManagedCursor.CURSOR_INTERNAL_PROPERTY_PREFIX; -import static org.apache.pulsar.broker.delayed.bucket.Bucket.DELIMITER; +import static org.apache.pulsar.broker.delayed.bucket.ImmutableBucket.DELIMITER; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Range; import com.google.common.collect.RangeMap; import com.google.common.collect.TreeRangeMap; import io.netty.util.Timeout; import io.netty.util.Timer; +import it.unimi.dsi.fastutil.longs.Long2ObjectMap; +import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; import java.time.Clock; import java.util.ArrayList; import java.util.Collections; @@ -34,7 +36,6 @@ import java.util.List; import java.util.Map; import java.util.NavigableSet; -import java.util.Optional; import java.util.TreeSet; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; @@ -53,7 +54,6 @@ import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.mutable.MutableLong; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.delayed.AbstractDelayedDeliveryTracker; import org.apache.pulsar.broker.delayed.DelayedDeliveryContext; @@ -94,7 +94,9 @@ public static record SnapshotKey(long ledgerId, long entryId) {} private final int maxNumBuckets; - private final AtomicLong numberDelayedMessages = new AtomicLong(0); + @Getter + @VisibleForTesting + private final BucketContext ctx; @Getter @@ -109,6 +111,10 @@ public static record SnapshotKey(long ledgerId, long entryId) {} @VisibleForTesting private final RangeMap immutableBuckets; + @Getter + @VisibleForTesting + private final BucketDelayedMessageIndex index = new BucketDelayedMessageIndex(); + @Getter @VisibleForTesting private final AtomicLong bucketsCount = new AtomicLong(0); @@ -165,15 +171,14 @@ public BucketDelayedDeliveryTracker(DelayedDeliveryContext context, this.sharedBucketPriorityQueue = new TripleLongPriorityQueue(); this.immutableBuckets = TreeRangeMap.create(); this.snapshotSegmentLastIndexMap = new ConcurrentHashMap<>(); - this.lastMutableBucket = - new MutableBucket(context.getName(), context.getCursor(), FutureUtil.Sequencer.create(), - bucketSnapshotStorage); + this.ctx = new BucketContext(context.getName(), context.getCursor(), FutureUtil.Sequencer.create(), + bucketSnapshotStorage); + this.lastMutableBucket = new MutableBucket(ctx); this.stats = new BucketDelayedMessageIndexStats(); // Close the tracker if failed to recover. try { - long recoveredMessages = recoverBucketSnapshot(); - this.numberDelayedMessages.set(recoveredMessages); + recoverBucketSnapshot(); } catch (RecoverDelayedDeliveryTrackerException e) { close(); throw e; @@ -181,25 +186,22 @@ public BucketDelayedDeliveryTracker(DelayedDeliveryContext context, } private synchronized long recoverBucketSnapshot() throws RecoverDelayedDeliveryTrackerException { - ManagedCursor cursor = this.lastMutableBucket.getCursor(); + ManagedCursor cursor = ctx.cursor(); Map cursorProperties = cursor.getCursorProperties(); if (MapUtils.isEmpty(cursorProperties)) { log.info("[{}] Recover delayed message index bucket snapshot finish, don't find bucket snapshot", context.getName()); return 0; } - FutureUtil.Sequencer sequencer = this.lastMutableBucket.getSequencer(); Map, ImmutableBucket> toBeDeletedBucketMap = new HashMap<>(); cursorProperties.keySet().forEach(key -> { if (key.startsWith(DELAYED_BUCKET_KEY_PREFIX)) { String[] keys = key.split(DELIMITER); checkArgument(keys.length == 3); ImmutableBucket immutableBucket = - new ImmutableBucket(context.getName(), cursor, sequencer, - this.lastMutableBucket.bucketSnapshotStorage, - Long.parseLong(keys[1]), Long.parseLong(keys[2])); - putAndCleanOverlapRange(Range.closed(immutableBucket.startLedgerId, immutableBucket.endLedgerId), - immutableBucket, toBeDeletedBucketMap); + new ImmutableBucket(ctx, Long.parseLong(keys[1]), Long.parseLong(keys[2])); + putAndCleanOverlapRange(Range.closed(immutableBucket.getStartLedgerId(), + immutableBucket.getEndLedgerId()), immutableBucket, toBeDeletedBucketMap); } }); @@ -255,19 +257,18 @@ private synchronized long recoverBucketSnapshot() throws RecoverDelayedDeliveryT immutableBucket.asyncDeleteBucketSnapshot(stats); } - MutableLong numberDelayedMessages = new MutableLong(0); long totalLength = 0; for (ImmutableBucket bucket : immutableBucketMap.values()) { - numberDelayedMessages.add(bucket.numberBucketDelayedMessages); + index.restore(bucket.getDelayedIndexBitMap()); totalLength += bucket.getSnapshotLength(); } totalSnapshotLengthBytes.set(totalLength); bucketsCount.set(immutableBuckets.asMapOfRanges().size()); log.info("[{}] Recover delayed message index bucket snapshot finish, buckets: {}, numberDelayedMessages: {}", - context.getName(), immutableBucketMap.size(), numberDelayedMessages.longValue()); + context.getName(), immutableBucketMap.size(), index.size()); - return numberDelayedMessages.getValue(); + return index.size(); } /** @@ -305,7 +306,7 @@ private synchronized void putAndCleanOverlapRange(Range range, ImmutableBu for (Map.Entry, ImmutableBucket> rangeEntry : subRangeMap.entrySet()) { // Use original key instead of truncated key for encloses check ImmutableBucket bucket = rangeEntry.getValue(); - Range originalKey = Range.closed(bucket.startLedgerId, bucket.endLedgerId); + Range originalKey = Range.closed(bucket.getStartLedgerId(), bucket.getEndLedgerId()); if (range.encloses(originalKey)) { toBeDeletedBucketMap.put(originalKey, bucket); @@ -332,19 +333,15 @@ public void run(Timeout timeout) throws Exception { super.run(timeout); } - private Optional findImmutableBucket(long ledgerId) { - if (immutableBuckets.asMapOfRanges().isEmpty()) { - return Optional.empty(); - } - - return Optional.ofNullable(immutableBuckets.get(ledgerId)); + private ImmutableBucket findImmutableBucket(long ledgerId) { + return immutableBuckets.get(ledgerId); } private void afterCreateImmutableBucket(Pair immutableBucketDelayedIndexPair, long startTime) { if (immutableBucketDelayedIndexPair != null) { ImmutableBucket immutableBucket = immutableBucketDelayedIndexPair.getLeft(); - putBucket(Range.closed(immutableBucket.startLedgerId, immutableBucket.endLedgerId), + putBucket(Range.closed(immutableBucket.getStartLedgerId(), immutableBucket.getEndLedgerId()), immutableBucket); DelayedIndex lastDelayedIndex = immutableBucketDelayedIndexPair.getRight(); @@ -387,9 +384,9 @@ private void afterCreateImmutableBucket(Pair immu immutableBucket.setSnapshotSegments(null); }); - immutableBucket.setCurrentSegmentEntryId(immutableBucket.lastSegmentEntryId); + immutableBucket.setCurrentSegmentEntryId(immutableBucket.getLastSegmentEntryId()); removeBucket( - Range.closed(immutableBucket.startLedgerId, immutableBucket.endLedgerId)); + Range.closed(immutableBucket.getStartLedgerId(), immutableBucket.getEndLedgerId())); snapshotSegmentLastIndexMap.remove( new SnapshotKey(lastDelayedIndex.getLedgerId(), lastDelayedIndex.getEntryId())); } @@ -403,6 +400,7 @@ private void afterCreateImmutableBucket(Pair immu @Override public synchronized boolean addMessage(long ledgerId, long entryId, long deliverAt) { if (deliverAt < 0 || deliverAt <= getCutoffTime()) { + removeIndexBit(ledgerId, entryId); return false; } @@ -410,7 +408,7 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver return true; } - boolean existBucket = findImmutableBucket(ledgerId).isPresent(); + boolean existBucket = findImmutableBucket(ledgerId) != null; // Create bucket snapshot if (!existBucket && ledgerId > lastMutableBucket.endLedgerId @@ -444,10 +442,8 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver // Message index belongs to previous bucket range or the current mutable bucket range, // enter sharedBucketPriorityQueue directly sharedBucketPriorityQueue.add(deliverAt, ledgerId, entryId); - lastMutableBucket.putIndexBit(ledgerId, entryId); } - - numberDelayedMessages.incrementAndGet(); + index.track(ledgerId, entryId); if (log.isDebugEnabled()) { log.debug("[{}] Add message {}:{} -- Delivery in {} ms ", context.getName(), ledgerId, entryId, @@ -473,10 +469,10 @@ synchronized List selectMergedBuckets(final List { // We should skip the bucket which last segment already been load to memory, // avoid record replicated index. - return bucket.lastSegmentEntryId > bucket.currentSegmentEntryId && !bucket.merging; + return bucket.getLastSegmentEntryId() > bucket.getCurrentSegmentEntryId() && !bucket.merging; })) { long numberMessages = immutableBuckets.stream() - .mapToLong(bucket -> bucket.numberBucketDelayedMessages) + .mapToLong(bucket -> bucket.getNumberBucketDelayedMessages()) .sum(); if (numberMessages <= minNumberMessages) { minNumberMessages = numberMessages; @@ -484,7 +480,8 @@ synchronized List selectMergedBuckets(final List bucket.firstScheduleTimestamps.get(bucket.currentSegmentEntryId)) + .mapToLong(bucket -> bucket.getFirstScheduleTimestamps() + .get(bucket.getCurrentSegmentEntryId())) .min().getAsLong(); if (scheduleTimestamp < minScheduleTimestamp) { minScheduleTimestamp = scheduleTimestamp; @@ -515,7 +512,7 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot() { return CompletableFuture.completedFuture(null); } - final String bucketsStr = toBeMergeImmutableBuckets.stream().map(Bucket::bucketKey).collect( + final String bucketsStr = toBeMergeImmutableBuckets.stream().map(ImmutableBucket::bucketKey).collect( Collectors.joining(",")).replaceAll(DELAYED_BUCKET_KEY_PREFIX + "_", ""); if (log.isDebugEnabled()) { log.info("[{}] Merging bucket snapshot, bucketKeys: {}", context.getName(), bucketsStr); @@ -558,13 +555,13 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(List>> getRemainFutures = - buckets.stream().map(ImmutableBucket::getRemainSnapshotSegment).toList(); + List>> getAllSnapshotFutures = + buckets.stream().map(ImmutableBucket::getAllSnapshotSegments).toList(); - return FutureUtil.waitForAll(getRemainFutures) + return FutureUtil.waitForAll(getAllSnapshotFutures) .thenApply(__ -> { return CombinedSegmentDelayedIndexQueue.wrap( - getRemainFutures.stream().map(CompletableFuture::join).toList()); + getAllSnapshotFutures.stream().map(CompletableFuture::join).toList()); }) .thenAccept(combinedDelayedIndexQueue -> { synchronized (BucketDelayedDeliveryTracker.this) { @@ -575,14 +572,14 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(List delayedIndexBitMap = - new HashMap<>(buckets.get(0).getDelayedIndexBitMap()); + Long2ObjectMap delayedIndexBitMap = + new Long2ObjectOpenHashMap<>(buckets.get(0).getDelayedIndexBitMap()); for (int i = 1; i < buckets.size(); i++) { - buckets.get(i).delayedIndexBitMap.forEach((ledgerId, bitMapB) -> { + buckets.get(i).getDelayedIndexBitMap().forEach((ledgerId, bitMapB) -> { delayedIndexBitMap.compute(ledgerId, (k, bitMap) -> { if (bitMap == null) { return bitMapB; @@ -607,7 +604,7 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(List getScheduledMessages(int maxMessages) long entryId = sharedBucketPriorityQueue.peekN3(); if (firstLiveLedgerId != null && ledgerId < firstLiveLedgerId) { sharedBucketPriorityQueue.pop(); - if (removeIndexBit(ledgerId, entryId)) { - numberDelayedMessages.decrementAndGet(); - } + removeIndexBit(ledgerId, entryId); continue; } if (timestamp > cutoffTime) { @@ -696,7 +691,7 @@ public synchronized NavigableSet getScheduledMessages(int maxMessages) break; } - final int preSegmentEntryId = bucket.currentSegmentEntryId; + final int preSegmentEntryId = bucket.getCurrentSegmentEntryId(); if (log.isDebugEnabled()) { log.debug("[{}] Loading next bucket snapshot segment, bucketKey: {}, nextSegmentEntryId: {}", context.getName(), bucket.bucketKey(), preSegmentEntryId + 1); @@ -715,7 +710,7 @@ public synchronized NavigableSet getScheduledMessages(int maxMessages) synchronized (BucketDelayedDeliveryTracker.this) { this.snapshotSegmentLastIndexMap.remove(snapshotKey); if (CollectionUtils.isEmpty(indexList)) { - removeBucket(Range.closed(bucket.startLedgerId, bucket.endLedgerId)); + removeBucket(Range.closed(bucket.getStartLedgerId(), bucket.getEndLedgerId())); bucket.asyncDeleteBucketSnapshot(stats); return; } @@ -741,7 +736,7 @@ public synchronized NavigableSet getScheduledMessages(int maxMessages) } else { log.info("[{}] Load next bucket snapshot segment finish, bucketKey: {}, segmentEntryId: {}", context.getName(), bucket.bucketKey(), - (preSegmentEntryId == bucket.lastSegmentEntryId) ? "-1" : preSegmentEntryId + 1); + (preSegmentEntryId == bucket.getLastSegmentEntryId()) ? "-1" : preSegmentEntryId + 1); stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.load, System.currentTimeMillis() - loadStartTime); @@ -759,13 +754,13 @@ public synchronized NavigableSet getScheduledMessages(int maxMessages) } } - positions.add(PositionFactory.create(ledgerId, entryId)); - sharedBucketPriorityQueue.pop(); - removeIndexBit(ledgerId, entryId); - - --n; - numberDelayedMessages.decrementAndGet(); + // Dedup: queue may carry the same position twice (initial seal + merge); only the + // first delivery of each position decrements the counter via removeIndexBit. + if (removeIndexBit(ledgerId, entryId)) { + positions.add(PositionFactory.create(ledgerId, entryId)); + --n; + } } updateTimer(); @@ -801,9 +796,9 @@ public synchronized CompletableFuture clear() { synchronized (BucketDelayedDeliveryTracker.this) { CompletableFuture future = cleanImmutableBuckets(); sharedBucketPriorityQueue.clear(); + index.clear(); lastMutableBucket.clear(); snapshotSegmentLastIndexMap.clear(); - numberDelayedMessages.set(0); return future; } }); @@ -840,7 +835,6 @@ private CompletableFuture cleanImmutableBuckets() { List> futures = new ArrayList<>(); bucketsToDelete.forEach((range, bucket) -> { removeBucket(range); - numberDelayedMessages.addAndGet(-bucket.getNumberBucketDelayedMessages()); futures.add(bucket.clear(stats)); }); @@ -848,21 +842,11 @@ private CompletableFuture cleanImmutableBuckets() { } private boolean removeIndexBit(long ledgerId, long entryId) { - if (lastMutableBucket.removeIndexBit(ledgerId, entryId)) { - return true; - } - - return findImmutableBucket(ledgerId).map(bucket -> bucket.removeIndexBit(ledgerId, entryId)) - .orElse(false); + return index.untrack(ledgerId, entryId); } public synchronized boolean containsMessage(long ledgerId, long entryId) { - if (lastMutableBucket.containsMessage(ledgerId, entryId)) { - return true; - } - - return findImmutableBucket(ledgerId).map(bucket -> bucket.containsMessage(ledgerId, entryId)) - .orElse(false); + return index.contains(ledgerId, entryId); } @@ -919,7 +903,8 @@ private CompletableFuture deleteBucketSnapshot(String ledgerName, synchronized (this) { snapshotSegmentLastIndexMap.entrySet().removeIf(entry -> entry.getValue() == bucket); removeBucket(range); - numberDelayedMessages.addAndGet(-bucket.getNumberBucketDelayedMessages()); + bucket.getDelayedIndexBitMap().forEach((ledgerId, bitmap) -> + bitmap.forEachLong(entryId -> index.untrack(ledgerId, entryId))); } return null; }); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndex.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndex.java new file mode 100644 index 0000000000000..86bb88e4b6ef8 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndex.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.delayed.bucket; + +import it.unimi.dsi.fastutil.longs.Long2ObjectMap; +import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; +import javax.annotation.concurrent.ThreadSafe; +import org.apache.pulsar.common.util.collections.LongBitmap; +import org.apache.pulsar.common.util.collections.LongBitmaps; + +/** + * Runtime truth for delayed messages that have been accepted but not yet delivered. + * Co-locates the bitmap and its cardinality so the counter is an invariant of the bitmap + * rather than a discipline callers must maintain. {@link ImmutableBucket#delayedIndexBitMap} + * is a frozen snapshot for BookKeeper writes/merge and is intentionally not consulted here. + */ +@ThreadSafe +final class BucketDelayedMessageIndex { + + private final Long2ObjectMap inflightIndex = new Long2ObjectOpenHashMap<>(); + private final AtomicLong size = new AtomicLong(0); + + /** Idempotent: re-tracking a position already in the index is a no-op. */ + void track(long ledgerId, long entryId) { + if (inflightIndex.computeIfAbsent(ledgerId, k -> LongBitmaps.create()).checkedAdd(entryId)) { + size.incrementAndGet(); + } + } + + /** @return true if the bit was present and removed; false if it was already absent. */ + boolean untrack(long ledgerId, long entryId) { + LongBitmap bitSet = inflightIndex.get(ledgerId); + if (bitSet == null || !bitSet.contains(entryId)) { + return false; + } + bitSet.remove(entryId); + if (bitSet.isEmpty()) { + inflightIndex.remove(ledgerId); + } + size.decrementAndGet(); + return true; + } + + boolean contains(long ledgerId, long entryId) { + LongBitmap bitSet = inflightIndex.get(ledgerId); + return bitSet != null && bitSet.contains(entryId); + } + + long size() { + return size.get(); + } + + void clear() { + inflightIndex.clear(); + size.set(0); + } + + /** + * Bulk-load after recovery. Built on {@link #track} so overlapping bits merge, not double-count. + */ + void restore(Map snapshot) { + snapshot.forEach((ledgerId, bitmap) -> + bitmap.forEachLong(entryId -> track(ledgerId, entryId))); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java index 57f48034cbeb4..e8cf822024740 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java @@ -19,29 +19,49 @@ package org.apache.pulsar.broker.delayed.bucket; import static org.apache.bookkeeper.mledger.util.Futures.executeWithRetry; +import static org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker.DELAYED_BUCKET_KEY_PREFIX; import static org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker.NULL_LONG_PROMISE; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; +import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; +import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.mutable.MutableLong; import org.apache.pulsar.broker.delayed.proto.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; -import org.apache.pulsar.broker.delayed.proto.SnapshotSegmentMetadata; +import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.LongBitmap; import org.apache.pulsar.common.util.collections.LongBitmaps; @Slf4j -class ImmutableBucket extends Bucket { +class ImmutableBucket { + + static final String DELIMITER = "_"; + static final int MaxRetryTimes = 3; + + private final BucketContext ctx; + + @Getter + private final long startLedgerId; + + @Getter + private final long endLedgerId; + + @Getter + @Setter + private Map delayedIndexBitMap = new Long2ObjectOpenHashMap<>(); @Setter private List snapshotSegments; @@ -49,11 +69,102 @@ class ImmutableBucket extends Bucket { boolean merging = false; @Setter + @Getter List firstScheduleTimestamps = new ArrayList<>(); - ImmutableBucket(String dispatcherName, ManagedCursor cursor, FutureUtil.Sequencer sequencer, - BucketSnapshotStorage storage, long startLedgerId, long endLedgerId) { - super(dispatcherName, cursor, sequencer, storage, startLedgerId, endLedgerId); + @Getter + @Setter + private long numberBucketDelayedMessages; + + @Getter + @Setter + private int lastSegmentEntryId; + + @Getter + @Setter + private volatile int currentSegmentEntryId; + + @Getter + @Setter + private volatile long snapshotLength; + + @Getter + @Setter + private volatile Long bucketId; + + @Getter + @Setter + private volatile CompletableFuture snapshotCreateFuture; + + ImmutableBucket(BucketContext ctx, long startLedgerId, long endLedgerId) { + this.ctx = ctx; + this.startLedgerId = startLedgerId; + this.endLedgerId = endLedgerId; + } + + String bucketKey() { + return String.join(DELIMITER, DELAYED_BUCKET_KEY_PREFIX, String.valueOf(startLedgerId), + String.valueOf(endLedgerId)); + } + + Optional> getSnapshotCreateFuture() { + return Optional.ofNullable(snapshotCreateFuture); + } + + Optional getBucketId() { + return Optional.ofNullable(bucketId); + } + + long getAndUpdateBucketId() { + Optional bucketIdOptional = getBucketId(); + if (bucketIdOptional.isPresent()) { + return bucketIdOptional.get(); + } + + String bucketIdStr = ctx.cursor().getCursorProperties().get(bucketKey()); + long bucketId = Long.parseLong(bucketIdStr); + setBucketId(bucketId); + return bucketId; + } + + CompletableFuture asyncSaveBucketSnapshot( + SnapshotMetadata snapshotMetadata, List bucketSnapshotSegments) { + final String bucketKey = bucketKey(); + final String cursorName = Codec.decode(ctx.cursor().getName()); + final String dispatcher = ctx.dispatcherName(); + final String topicName = dispatcher.substring(0, dispatcher.lastIndexOf(" / " + cursorName)); + return executeWithRetry( + () -> ctx.bucketSnapshotStorage().createBucketSnapshot(snapshotMetadata, bucketSnapshotSegments, + bucketKey, topicName, cursorName) + .whenComplete((__, ex) -> { + if (ex != null) { + log.warn("[{}] Failed to create bucket snapshot, bucketKey: {}", + dispatcher, bucketKey, ex); + } + }), BucketSnapshotPersistenceException.class, MaxRetryTimes).thenCompose(newBucketId -> { + setBucketId(newBucketId); + + return putBucketKeyId(bucketKey, newBucketId).exceptionally(ex -> { + log.warn("[{}] Failed to record bucketId {} to cursor property, bucketKey: {}", + dispatcher, newBucketId, bucketKey, ex); + return null; + }).thenApply(__ -> newBucketId); + }); + } + + private CompletableFuture putBucketKeyId(String bucketKey, Long bucketId) { + if (bucketId == null) { + return FutureUtil.failedFuture(new NullPointerException("Expected bucketId should not be null")); + } + return ctx.sequencer().sequential(() -> + executeWithRetry(() -> ctx.cursor().putCursorProperty(bucketKey, String.valueOf(bucketId)), + ManagedLedgerException.BadVersionException.class, MaxRetryTimes)); + } + + CompletableFuture removeBucketCursorProperty(String bucketKey) { + return ctx.sequencer().sequential(() -> + executeWithRetry(() -> ctx.cursor().removeCursorProperty(bucketKey), + ManagedLedgerException.BadVersionException.class, MaxRetryTimes)); } public Optional> getSnapshotSegments() { @@ -76,29 +187,31 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b final long cutoffTime = cutoffTimeSupplier.get(); // Load Metadata of bucket snapshot final String bucketKey = bucketKey(); - loadMetaDataFuture = executeWithRetry(() -> bucketSnapshotStorage.getBucketSnapshotMetadata(bucketId) + loadMetaDataFuture = executeWithRetry(() -> ctx.bucketSnapshotStorage().getBucketSnapshotMetadata(bucketId) .whenComplete((___, ex) -> { if (ex != null) { - log.warn("[{}] Failed to get bucket snapshot metadata," - + " bucketKey: {}, bucketId: {}", - dispatcherName, bucketKey, bucketId, ex); + log.warn("[{}] Failed to get bucket snapshot metadata, bucketKey: {}, bucketId: {}", + ctx.dispatcherName(), bucketKey, bucketId, ex); } }), BucketSnapshotPersistenceException.class, MaxRetryTimes) .thenApply(snapshotMetadata -> { - List metadataList = - snapshotMetadata.getMetadataListList(); + int metadataListSize = snapshotMetadata.getMetadataListCount(); // Skip all already reach schedule time snapshot segments int nextSnapshotEntryIndex = 0; - while (nextSnapshotEntryIndex < metadataList.size() - && metadataList.get(nextSnapshotEntryIndex).getMaxScheduleTimestamp() <= cutoffTime) { + while (nextSnapshotEntryIndex < metadataListSize + && snapshotMetadata.getMetadataList(nextSnapshotEntryIndex) + .getMaxScheduleTimestamp() <= cutoffTime) { nextSnapshotEntryIndex++; } - this.setLastSegmentEntryId(metadataList.size()); - this.recoverDelayedIndexBitMapAndNumber(nextSnapshotEntryIndex, metadataList); - List firstScheduleTimestamps = metadataList.stream().map( - SnapshotSegmentMetadata::getMinScheduleTimestamp).toList(); + this.setLastSegmentEntryId(metadataListSize); + this.recoverDelayedIndexBitMapAndNumber(nextSnapshotEntryIndex, snapshotMetadata); + List firstScheduleTimestamps = new ArrayList<>(); + for (int i = 0; i < metadataListSize; i++) { + firstScheduleTimestamps.add( + snapshotMetadata.getMetadataList(i).getMinScheduleTimestamp()); + } this.setFirstScheduleTimestamps(firstScheduleTimestamps); return nextSnapshotEntryIndex + 1; @@ -113,12 +226,12 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b } return executeWithRetry( - () -> bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, nextSegmentEntryId, + () -> ctx.bucketSnapshotStorage().getBucketSnapshotSegment(bucketId, nextSegmentEntryId, nextSegmentEntryId).whenComplete((___, ex) -> { if (ex != null) { - log.warn("[{}] Failed to get bucket snapshot segment. bucketKey: {}," - + " bucketId: {}, segmentEntryId: {}", dispatcherName, bucketKey(), - bucketId, nextSegmentEntryId, ex); + log.warn("[{}] Failed to get bucket snapshot segment, bucketKey: {}, bucketId: {}," + + " segmentEntryId: {}", ctx.dispatcherName(), + bucketKey(), bucketId, nextSegmentEntryId, ex); } }), BucketSnapshotPersistenceException.class, MaxRetryTimes) .thenCompose(bucketSnapshotSegments -> { @@ -144,13 +257,11 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b * Recover delayed index bit map and message numbers. */ private void recoverDelayedIndexBitMapAndNumber(int startSnapshotIndex, - List segmentMetaList) { + SnapshotMetadata snapshotMetadata) { delayedIndexBitMap.clear(); // cleanup dirty bm final var numberMessages = new MutableLong(0); - for (int i = startSnapshotIndex; i < segmentMetaList.size(); i++) { - for (final var entry : segmentMetaList.get(i).getDelayedIndexBitMapMap().entrySet()) { - final var ledgerId = entry.getKey(); - final var bs = entry.getValue(); + for (int i = startSnapshotIndex; i < snapshotMetadata.getMetadataListCount(); i++) { + snapshotMetadata.getMetadataList(i).getDelayedIndexBitMap().forEach((ledgerId, bs) -> { final ByteBuf buf = Unpooled.wrappedBuffer(bs.asReadOnlyByteBuffer()); try { final LongBitmap sbm = LongBitmaps.deserialize(buf); @@ -165,24 +276,22 @@ private void recoverDelayedIndexBitMapAndNumber(int startSnapshotIndex, } finally { buf.release(); } - } + }); } setNumberBucketDelayedMessages(numberMessages.longValue()); } - CompletableFuture> getRemainSnapshotSegment() { - int nextSegmentEntryId = currentSegmentEntryId + 1; - if (nextSegmentEntryId > lastSegmentEntryId) { + CompletableFuture> getAllSnapshotSegments() { + if (lastSegmentEntryId < 1) { return CompletableFuture.completedFuture(Collections.emptyList()); } return executeWithRetry(() -> { - return bucketSnapshotStorage.getBucketSnapshotSegment(getAndUpdateBucketId(), nextSegmentEntryId, + return ctx.bucketSnapshotStorage().getBucketSnapshotSegment(getAndUpdateBucketId(), 1, lastSegmentEntryId).whenComplete((__, ex) -> { if (ex != null) { - log.warn( - "[{}] Failed to get remain bucket snapshot segment, bucketKey: {}," - + " nextSegmentEntryId: {}, lastSegmentEntryId: {}", - dispatcherName, bucketKey(), nextSegmentEntryId, lastSegmentEntryId, ex); + log.warn("[{}] Failed to get all bucket snapshot segments for merge, bucketKey: {}," + + " lastSegmentEntryId: {}", ctx.dispatcherName(), bucketKey(), + lastSegmentEntryId, ex); } }); }, BucketSnapshotPersistenceException.class, MaxRetryTimes); @@ -194,17 +303,17 @@ CompletableFuture asyncDeleteBucketSnapshot(BucketDelayedMessageIndexStats String bucketKey = bucketKey(); long bucketId = getAndUpdateBucketId(); - return executeWithRetry(() -> bucketSnapshotStorage.deleteBucketSnapshot(bucketId), + return executeWithRetry(() -> ctx.bucketSnapshotStorage().deleteBucketSnapshot(bucketId), BucketSnapshotPersistenceException.class, MaxRetryTimes) .whenComplete((__, ex) -> { if (ex != null) { log.error("[{}] Failed to delete bucket snapshot, bucketId: {}, bucketKey: {}", - dispatcherName, bucketId, bucketKey, ex); + ctx.dispatcherName(), bucketId, bucketKey, ex); stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.delete); } else { log.info("[{}] Delete bucket snapshot finish, bucketId: {}, bucketKey: {}", - dispatcherName, bucketId, bucketKey); + ctx.dispatcherName(), bucketId, bucketKey); stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.delete, System.currentTimeMillis() - deleteStartTime); @@ -221,10 +330,10 @@ CompletableFuture clear(BucketDelayedMessageIndexStats stats) { protected CompletableFuture asyncUpdateSnapshotLength() { long bucketId = getAndUpdateBucketId(); - return bucketSnapshotStorage.getBucketSnapshotLength(bucketId).whenComplete((length, ex) -> { + return ctx.bucketSnapshotStorage().getBucketSnapshotLength(bucketId).whenComplete((length, ex) -> { if (ex != null) { log.error("[{}] Failed to get snapshot length, bucketId: {}, bucketKey: {}", - dispatcherName, bucketId, bucketKey(), ex); + ctx.dispatcherName(), bucketId, bucketKey(), ex); } }); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java index 02f22f5b17704..d0a9e8aef1a1b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java @@ -27,25 +27,27 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.delayed.proto.DelayedIndex; import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; import org.apache.pulsar.broker.delayed.proto.SnapshotSegmentMetadata; -import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.LongBitmap; import org.apache.pulsar.common.util.collections.LongBitmaps; import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue; @Slf4j -class MutableBucket extends Bucket implements AutoCloseable { +class MutableBucket implements AutoCloseable { + + private final BucketContext ctx; private final TripleLongPriorityQueue priorityQueue; - MutableBucket(String dispatcherName, ManagedCursor cursor, FutureUtil.Sequencer sequencer, - BucketSnapshotStorage bucketSnapshotStorage) { - super(dispatcherName, cursor, sequencer, bucketSnapshotStorage, -1L, -1L); + long startLedgerId = -1L; + long endLedgerId = -1L; + + MutableBucket(BucketContext ctx) { + this.ctx = ctx; this.priorityQueue = new TripleLongPriorityQueue(); } @@ -63,10 +65,9 @@ Pair createImmutableBucketAndAsyncPersistent( TripleLongPriorityQueue sharedQueue, DelayedIndexQueue delayedIndexQueue, final long startLedgerId, final long endLedgerId) { if (log.isDebugEnabled()) { - log.debug("[{}] Creating bucket snapshot, startLedgerId: {}, endLedgerId: {}", dispatcherName, + log.debug("[{}] Creating bucket snapshot, startLedgerId: {}, endLedgerId: {}", ctx.dispatcherName(), startLedgerId, endLedgerId); } - if (delayedIndexQueue.isEmpty()) { return null; } @@ -97,8 +98,6 @@ Pair createImmutableBucketAndAsyncPersistent( final long ledgerId = delayedIndex.getLedgerId(); final long entryId = delayedIndex.getEntryId(); - removeIndexBit(ledgerId, entryId); - checkArgument(ledgerId >= startLedgerId && ledgerId <= endLedgerId); // Move first segment of bucket snapshot to sharedBucketPriorityQueue @@ -122,8 +121,7 @@ Pair createImmutableBucketAndAsyncPersistent( final var entry = iterator.next(); final var lId = entry.getKey(); final var bm = entry.getValue(); - segmentMetadataBuilder.putDelayedIndexBitMap(lId, - UnsafeByteOperations.unsafeWrap(bm.serialize())); + segmentMetadataBuilder.putDelayedIndexBitMap(lId, UnsafeByteOperations.unsafeWrap(bm.serialize())); immutableBucketBitMap.compute(lId, (__, bm0) -> { if (bm0 == null) { return bm; @@ -135,7 +133,7 @@ Pair createImmutableBucketAndAsyncPersistent( } segmentMetadataList.add(segmentMetadataBuilder.build()); - segmentMetadataBuilder.clear(); + segmentMetadataBuilder = SnapshotSegmentMetadata.newBuilder(); bucketSnapshotSegments.add(snapshotSegment); snapshotSegment = new SnapshotSegment(); @@ -148,8 +146,7 @@ Pair createImmutableBucketAndAsyncPersistent( final int lastSegmentEntryId = segmentMetadataList.size(); - ImmutableBucket bucket = new ImmutableBucket(dispatcherName, cursor, sequencer, bucketSnapshotStorage, - startLedgerId, endLedgerId); + ImmutableBucket bucket = new ImmutableBucket(ctx, startLedgerId, endLedgerId); bucket.setCurrentSegmentEntryId(1); bucket.setNumberBucketDelayedMessages(numMessages); bucket.setLastSegmentEntryId(lastSegmentEntryId); @@ -166,7 +163,7 @@ Pair createImmutableBucketAndAsyncPersistent( DelayedIndex lastDelayedIndex = firstSnapshotSegment.getIndexeAt(firstSnapshotSegment.getIndexesCount() - 1); Pair result = Pair.of(bucket, lastDelayedIndex); - CompletableFuture future = asyncSaveBucketSnapshot(bucket, + CompletableFuture future = bucket.asyncSaveBucketSnapshot( bucketSnapshotMetadata, bucketSnapshotSegments); bucket.setSnapshotCreateFuture(future); @@ -195,7 +192,6 @@ void resetLastMutableBucketRange() { void clear() { this.resetLastMutableBucketRange(); - this.delayedIndexBitMap.clear(); this.priorityQueue.clear(); } @@ -225,6 +221,5 @@ void addMessage(long ledgerId, long entryId, long deliverAt) { this.startLedgerId = ledgerId; } this.endLedgerId = ledgerId; - putIndexBit(ledgerId, entryId); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java index f26204991d955..9284c08c8c120 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java @@ -149,7 +149,8 @@ public Object[][] provider(Method method) throws Exception { new BucketDelayedDeliveryTracker(dispatcher, timer, 500, clock, true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 50) }}; - case "testMergeSnapshot", "testWithBkException", "testWithCreateFailDowngrade" -> new Object[][]{{ + case "testMergeSnapshot", "testWithBkException", "testWithCreateFailDowngrade", + "testMergePreservesAllSnapshotSegments" -> new Object[][]{{ new BucketDelayedDeliveryTracker(dispatcher, timer, 100000, clock, true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 10) }}; @@ -157,7 +158,12 @@ public Object[][] provider(Method method) throws Exception { new BucketDelayedDeliveryTracker(dispatcher, timer, 100000, clock, true, bucketSnapshotStorage, 20, TimeUnit.HOURS.toMillis(1), 5, 100) }}; - case "testExpiredTrackedMessageReturnsFalse", "testRecoverThenExpireAddMessage" -> new Object[][]{{ + case "testClear" -> new Object[][]{{ + new BucketDelayedDeliveryTracker(dispatcher, timer, 100000, clock, + true, bucketSnapshotStorage, 1000, TimeUnit.MILLISECONDS.toMillis(100), -1, 50) + }}; + case "testExpiredTrackedMessageReturnsFalse", "testRecoverThenExpireAddMessage", + "testExpiredTrackedMessageDecrementsCount" -> new Object[][]{{ new BucketDelayedDeliveryTracker(dispatcher, timer, 1, clock, true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 50) }}; @@ -235,6 +241,40 @@ public void testRecoverThenExpireAddMessage(BucketDelayedDeliveryTracker tracker tracker2.addMessage(1, 1, 1000)); } + @Test(dataProvider = "delayedTracker") + public void testExpiredTrackedMessageDecrementsCount(BucketDelayedDeliveryTracker tracker) { + clockTime.set(1000); + tracker.addMessage(1, 1, 2000); + assertEquals(tracker.getNumberOfDelayedMessages(), 1); + + clockTime.set(2500); + assertFalse(tracker.addMessage(1, 1, 2000)); + assertEquals(tracker.getNumberOfDelayedMessages(), 0); + assertFalse(tracker.containsMessage(1, 1)); + tracker.close(); + } + + @Test(dataProvider = "delayedTracker") + public void testMergePreservesAllSnapshotSegments(BucketDelayedDeliveryTracker tracker) throws Exception { + clockTime.set(0); + for (int i = 1; i <= 56; i++) { + tracker.addMessage(i, i, i * 10); + } + Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> + assertTrue(tracker.getImmutableBuckets().asMapOfRanges().values().stream() + .noneMatch(x -> x.merging))); + assertEquals(tracker.getNumberOfDelayedMessages(), 56); + + tracker.close(); + clockTime.set(0); + BucketDelayedDeliveryTracker tracker2 = new BucketDelayedDeliveryTracker( + dispatcher, timer, 100000, clock, + true, bucketSnapshotStorage, 5, TimeUnit.MILLISECONDS.toMillis(10), -1, 10); + + assertEquals(tracker2.getNumberOfDelayedMessages(), 55); + tracker2.close(); + } + @Test(dataProvider = "delayedTracker", invocationCount = 10) public void testRecoverSnapshot(BucketDelayedDeliveryTracker tracker) throws Exception { for (int i = 1; i <= 100; i++) { @@ -359,7 +399,7 @@ public void testMergeSnapshot(final BucketDelayedDeliveryTracker tracker) throws clockTime.set(110 * 10); NavigableSet scheduledMessages = new TreeSet<>(); - Awaitility.await().untilAsserted(() -> { + Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { scheduledMessages.addAll(tracker2.getScheduledMessages(110)); assertEquals(scheduledMessages.size(), 110); }); @@ -435,7 +475,7 @@ public void testWithBkException(final BucketDelayedDeliveryTracker tracker) thro assertEquals(tracker2.getScheduledMessages(100).size(), 0); Set scheduledMessages = new TreeSet<>(); - Awaitility.await().untilAsserted(() -> { + Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { scheduledMessages.addAll(tracker2.getScheduledMessages(100)); assertEquals(scheduledMessages.size(), delayedMessagesInSnapshotValue); }); @@ -560,9 +600,7 @@ public CompletableFuture deleteBucketSnapshot(long bucketId) { private ImmutableBucket createMergeableBucket(TrackerWithStorage trackerWithStorage, long startLedgerId, long endLedgerId, List firstScheduleTimestamps) { - MutableBucket mutableBucket = trackerWithStorage.tracker.getLastMutableBucket(); - ImmutableBucket bucket = new ImmutableBucket(mutableBucket.dispatcherName, mutableBucket.cursor, - mutableBucket.sequencer, mutableBucket.bucketSnapshotStorage, startLedgerId, endLedgerId); + ImmutableBucket bucket = new ImmutableBucket(trackerWithStorage.tracker.getCtx(), startLedgerId, endLedgerId); bucket.setCurrentSegmentEntryId(1); bucket.setLastSegmentEntryId(firstScheduleTimestamps.size()); bucket.setFirstScheduleTimestamps(firstScheduleTimestamps); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexTest.java new file mode 100644 index 0000000000000..57774605580f1 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexTest.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.delayed.bucket; + +import static org.assertj.core.api.Assertions.assertThat; +import it.unimi.dsi.fastutil.longs.Long2ObjectMap; +import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; +import org.apache.pulsar.common.util.collections.LongBitmap; +import org.apache.pulsar.common.util.collections.LongBitmaps; +import org.testng.annotations.Test; + +public class BucketDelayedMessageIndexTest { + + @Test + public void trackThenContains() { + BucketDelayedMessageIndex index = new BucketDelayedMessageIndex(); + + index.track(7L, 100L); + + assertThat(index.contains(7L, 100L)).isTrue(); + assertThat(index.contains(7L, 101L)).isFalse(); + assertThat(index.contains(8L, 100L)).isFalse(); + assertThat(index.size()).isEqualTo(1L); + } + + @Test + public void untrackReturnsTrueFirstTimeAndFalseAfter() { + BucketDelayedMessageIndex index = new BucketDelayedMessageIndex(); + index.track(1L, 1L); + + assertThat(index.untrack(1L, 1L)).isTrue(); + assertThat(index.size()).isZero(); + + assertThat(index.untrack(1L, 1L)).isFalse(); + assertThat(index.size()).isZero(); + } + + @Test + public void untrackOnAbsentBitIsSafe() { + BucketDelayedMessageIndex index = new BucketDelayedMessageIndex(); + + assertThat(index.untrack(99L, 99L)).isFalse(); + assertThat(index.size()).isZero(); + assertThat(index.contains(99L, 99L)).isFalse(); + } + + @Test + public void trackIsIdempotent() { + BucketDelayedMessageIndex index = new BucketDelayedMessageIndex(); + + index.track(3L, 5L); + index.track(3L, 5L); + index.track(3L, 5L); + + assertThat(index.size()).isEqualTo(1L); + assertThat(index.contains(3L, 5L)).isTrue(); + } + + @Test + public void trackAcrossManyLedgersKeepsCounterCorrect() { + BucketDelayedMessageIndex index = new BucketDelayedMessageIndex(); + + for (long ledger = 1; ledger <= 5; ledger++) { + for (long entry = 1; entry <= 10; entry++) { + index.track(ledger, entry); + } + } + + assertThat(index.size()).isEqualTo(50L); + + // Drain half. + for (long ledger = 1; ledger <= 5; ledger++) { + for (long entry = 1; entry <= 5; entry++) { + assertThat(index.untrack(ledger, entry)).isTrue(); + } + } + assertThat(index.size()).isEqualTo(25L); + } + + @Test + public void clearResetsBitmapAndCounter() { + BucketDelayedMessageIndex index = new BucketDelayedMessageIndex(); + index.track(1L, 1L); + index.track(2L, 2L); + assertThat(index.size()).isEqualTo(2L); + + index.clear(); + + assertThat(index.size()).isZero(); + assertThat(index.contains(1L, 1L)).isFalse(); + assertThat(index.contains(2L, 2L)).isFalse(); + + // Index remains usable after clear. + index.track(3L, 3L); + assertThat(index.size()).isEqualTo(1L); + assertThat(index.contains(3L, 3L)).isTrue(); + } + + @Test + public void restoreLoadsBitsFromSnapshot() { + BucketDelayedMessageIndex index = new BucketDelayedMessageIndex(); + + Long2ObjectMap snapshot = new Long2ObjectOpenHashMap<>(); + LongBitmap ledger1 = LongBitmaps.create(); + ledger1.add(10L); + ledger1.add(11L); + snapshot.put(1L, ledger1); + LongBitmap ledger2 = LongBitmaps.create(); + ledger2.add(20L); + snapshot.put(2L, ledger2); + + index.restore(snapshot); + + assertThat(index.size()).isEqualTo(3L); + assertThat(index.contains(1L, 10L)).isTrue(); + assertThat(index.contains(1L, 11L)).isTrue(); + assertThat(index.contains(2L, 20L)).isTrue(); + } + + @Test + public void restoreIsIdempotentOnOverlappingSnapshots() { + BucketDelayedMessageIndex index = new BucketDelayedMessageIndex(); + + Long2ObjectMap firstBucket = new Long2ObjectOpenHashMap<>(); + LongBitmap bits = LongBitmaps.create(); + bits.add(5L); + bits.add(6L); + firstBucket.put(7L, bits); + + Long2ObjectMap secondBucket = new Long2ObjectOpenHashMap<>(); + LongBitmap overlapping = LongBitmaps.create(); + overlapping.add(5L); // overlap with firstBucket + overlapping.add(8L); + secondBucket.put(7L, overlapping); + + index.restore(firstBucket); + index.restore(secondBucket); + + assertThat(index.size()).isEqualTo(3L); // 5, 6, 8 — not 4 + assertThat(index.contains(7L, 5L)).isTrue(); + assertThat(index.contains(7L, 6L)).isTrue(); + assertThat(index.contains(7L, 8L)).isTrue(); + } + + @Test + public void restoreAfterTrackMergesCorrectly() { + BucketDelayedMessageIndex index = new BucketDelayedMessageIndex(); + index.track(1L, 1L); + assertThat(index.size()).isEqualTo(1L); + + Long2ObjectMap snapshot = new Long2ObjectOpenHashMap<>(); + LongBitmap bits = LongBitmaps.create(); + bits.add(1L); // overlap with the existing tracked bit + bits.add(2L); + snapshot.put(1L, bits); + + index.restore(snapshot); + + assertThat(index.size()).isEqualTo(2L); + assertThat(index.contains(1L, 1L)).isTrue(); + assertThat(index.contains(1L, 2L)).isTrue(); + } +} From 95a0edc962c3cfd22b595600bb7f464efc7292f0 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Fri, 28 Aug 2026 15:48:54 +0800 Subject: [PATCH 202/213] [improve][broker] Unify dedup index for in-memory and bucket delayed delivery trackers Promote the bucket-only dedup index introduced by #26251 to the shared delayed package as DelayedMessageIndex and use it in the in-memory tracker as well. addMessage now tracks positions idempotently, so a duplicate position is no longer enqueued twice (previously it would be scheduled and delivered twice). The manually maintained delayedMessagesCount counter is replaced by index.size(), making the counter an invariant of the bitmap instead of a discipline at every mutation site. --- ...ageIndex.java => DelayedMessageIndex.java} | 29 ++++++++++++------- .../InMemoryDelayedDeliveryTracker.java | 20 +++++++------ .../bucket/BucketDelayedDeliveryTracker.java | 3 +- ...Test.java => DelayedMessageIndexTest.java} | 22 +++++++------- .../delayed/InMemoryDeliveryTrackerTest.java | 19 ++++++++++++ 5 files changed, 61 insertions(+), 32 deletions(-) rename pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/{bucket/BucketDelayedMessageIndex.java => DelayedMessageIndex.java} (75%) rename pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/{bucket/BucketDelayedMessageIndexTest.java => DelayedMessageIndexTest.java} (87%) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndex.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedMessageIndex.java similarity index 75% rename from pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndex.java rename to pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedMessageIndex.java index 86bb88e4b6ef8..3eb05bae5c3fc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndex.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/DelayedMessageIndex.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.broker.delayed.bucket; +package org.apache.pulsar.broker.delayed; import it.unimi.dsi.fastutil.longs.Long2ObjectMap; import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; @@ -29,24 +29,31 @@ /** * Runtime truth for delayed messages that have been accepted but not yet delivered. * Co-locates the bitmap and its cardinality so the counter is an invariant of the bitmap - * rather than a discipline callers must maintain. {@link ImmutableBucket#delayedIndexBitMap} - * is a frozen snapshot for BookKeeper writes/merge and is intentionally not consulted here. + * rather than a discipline callers must maintain. Used by both the in-memory and the bucket + * delayed delivery trackers; the bucket trackers' frozen snapshot bitmaps for BookKeeper + * writes/merge are intentionally not consulted here. */ @ThreadSafe -final class BucketDelayedMessageIndex { +public final class DelayedMessageIndex { private final Long2ObjectMap inflightIndex = new Long2ObjectOpenHashMap<>(); private final AtomicLong size = new AtomicLong(0); - /** Idempotent: re-tracking a position already in the index is a no-op. */ - void track(long ledgerId, long entryId) { + /** + * Idempotent: re-tracking a position already in the index is a no-op. + * + * @return true if the position was newly tracked, false if it was already present + */ + public boolean track(long ledgerId, long entryId) { if (inflightIndex.computeIfAbsent(ledgerId, k -> LongBitmaps.create()).checkedAdd(entryId)) { size.incrementAndGet(); + return true; } + return false; } /** @return true if the bit was present and removed; false if it was already absent. */ - boolean untrack(long ledgerId, long entryId) { + public boolean untrack(long ledgerId, long entryId) { LongBitmap bitSet = inflightIndex.get(ledgerId); if (bitSet == null || !bitSet.contains(entryId)) { return false; @@ -59,16 +66,16 @@ boolean untrack(long ledgerId, long entryId) { return true; } - boolean contains(long ledgerId, long entryId) { + public boolean contains(long ledgerId, long entryId) { LongBitmap bitSet = inflightIndex.get(ledgerId); return bitSet != null && bitSet.contains(entryId); } - long size() { + public long size() { return size.get(); } - void clear() { + public void clear() { inflightIndex.clear(); size.set(0); } @@ -76,7 +83,7 @@ void clear() { /** * Bulk-load after recovery. Built on {@link #track} so overlapping bits merge, not double-count. */ - void restore(Map snapshot) { + public void restore(Map snapshot) { snapshot.forEach((ledgerId, bitmap) -> bitmap.forEachLong(entryId -> track(ledgerId, entryId))); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java index fc4fe56b0f17c..b5e6277064cda 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/InMemoryDelayedDeliveryTracker.java @@ -24,7 +24,6 @@ import java.util.NavigableSet; import java.util.TreeSet; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.atomic.AtomicLong; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Position; @@ -53,9 +52,11 @@ public class InMemoryDelayedDeliveryTracker extends AbstractDelayedDeliveryTrack // Track whether we have seen all messages with fixed delay so far. private boolean messagesHaveFixedDelay = true; - // Count of delayed messages in the tracker, maintained incrementally so that stats reads - // do not contend with mutation paths (#24430 / #25990). - private final AtomicLong delayedMessagesCount = new AtomicLong(0); + // Dedup index of in-flight delayed messages; the counter is an invariant of the bitmap + // so stats reads stay lock-free and duplicate positions are tracked once (#26251). + @Getter + @VisibleForTesting + private final DelayedMessageIndex index = new DelayedMessageIndex(); InMemoryDelayedDeliveryTracker(AbstractPersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis, @@ -94,8 +95,9 @@ public boolean addMessage(long ledgerId, long entryId, long deliverAt) { deliverAt - clock.millis()); } - priorityQueue.add(deliverAt, ledgerId, entryId); - delayedMessagesCount.incrementAndGet(); + if (index.track(ledgerId, entryId)) { + priorityQueue.add(deliverAt, ledgerId, entryId); + } updateTimer(); checkAndUpdateHighest(deliverAt); @@ -147,7 +149,7 @@ public NavigableSet getScheduledMessages(int maxMessages) { positions.add(PositionFactory.create(ledgerId, entryId)); priorityQueue.pop(); - delayedMessagesCount.decrementAndGet(); + index.untrack(ledgerId, entryId); --n; } @@ -168,13 +170,13 @@ public NavigableSet getScheduledMessages(int maxMessages) { @Override public CompletableFuture clear() { this.priorityQueue.clear(); - this.delayedMessagesCount.set(0); + this.index.clear(); return CompletableFuture.completedFuture(null); } @Override public long getNumberOfDelayedMessages() { - return delayedMessagesCount.get(); + return index.size(); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index cd009cc63e3b1..471cbd7272a26 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -57,6 +57,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.delayed.AbstractDelayedDeliveryTracker; import org.apache.pulsar.broker.delayed.DelayedDeliveryContext; +import org.apache.pulsar.broker.delayed.DelayedMessageIndex; import org.apache.pulsar.broker.delayed.DispatcherDelayedDeliveryContext; import org.apache.pulsar.broker.delayed.proto.DelayedIndex; import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; @@ -113,7 +114,7 @@ public static record SnapshotKey(long ledgerId, long entryId) {} @Getter @VisibleForTesting - private final BucketDelayedMessageIndex index = new BucketDelayedMessageIndex(); + private final DelayedMessageIndex index = new DelayedMessageIndex(); @Getter @VisibleForTesting diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/DelayedMessageIndexTest.java similarity index 87% rename from pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexTest.java rename to pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/DelayedMessageIndexTest.java index 57774605580f1..28a245d0af0b5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedMessageIndexTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/DelayedMessageIndexTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.broker.delayed.bucket; +package org.apache.pulsar.broker.delayed; import static org.assertj.core.api.Assertions.assertThat; import it.unimi.dsi.fastutil.longs.Long2ObjectMap; @@ -25,11 +25,11 @@ import org.apache.pulsar.common.util.collections.LongBitmaps; import org.testng.annotations.Test; -public class BucketDelayedMessageIndexTest { +public class DelayedMessageIndexTest { @Test public void trackThenContains() { - BucketDelayedMessageIndex index = new BucketDelayedMessageIndex(); + DelayedMessageIndex index = new DelayedMessageIndex(); index.track(7L, 100L); @@ -41,7 +41,7 @@ public void trackThenContains() { @Test public void untrackReturnsTrueFirstTimeAndFalseAfter() { - BucketDelayedMessageIndex index = new BucketDelayedMessageIndex(); + DelayedMessageIndex index = new DelayedMessageIndex(); index.track(1L, 1L); assertThat(index.untrack(1L, 1L)).isTrue(); @@ -53,7 +53,7 @@ public void untrackReturnsTrueFirstTimeAndFalseAfter() { @Test public void untrackOnAbsentBitIsSafe() { - BucketDelayedMessageIndex index = new BucketDelayedMessageIndex(); + DelayedMessageIndex index = new DelayedMessageIndex(); assertThat(index.untrack(99L, 99L)).isFalse(); assertThat(index.size()).isZero(); @@ -62,7 +62,7 @@ public void untrackOnAbsentBitIsSafe() { @Test public void trackIsIdempotent() { - BucketDelayedMessageIndex index = new BucketDelayedMessageIndex(); + DelayedMessageIndex index = new DelayedMessageIndex(); index.track(3L, 5L); index.track(3L, 5L); @@ -74,7 +74,7 @@ public void trackIsIdempotent() { @Test public void trackAcrossManyLedgersKeepsCounterCorrect() { - BucketDelayedMessageIndex index = new BucketDelayedMessageIndex(); + DelayedMessageIndex index = new DelayedMessageIndex(); for (long ledger = 1; ledger <= 5; ledger++) { for (long entry = 1; entry <= 10; entry++) { @@ -95,7 +95,7 @@ public void trackAcrossManyLedgersKeepsCounterCorrect() { @Test public void clearResetsBitmapAndCounter() { - BucketDelayedMessageIndex index = new BucketDelayedMessageIndex(); + DelayedMessageIndex index = new DelayedMessageIndex(); index.track(1L, 1L); index.track(2L, 2L); assertThat(index.size()).isEqualTo(2L); @@ -114,7 +114,7 @@ public void clearResetsBitmapAndCounter() { @Test public void restoreLoadsBitsFromSnapshot() { - BucketDelayedMessageIndex index = new BucketDelayedMessageIndex(); + DelayedMessageIndex index = new DelayedMessageIndex(); Long2ObjectMap snapshot = new Long2ObjectOpenHashMap<>(); LongBitmap ledger1 = LongBitmaps.create(); @@ -135,7 +135,7 @@ public void restoreLoadsBitsFromSnapshot() { @Test public void restoreIsIdempotentOnOverlappingSnapshots() { - BucketDelayedMessageIndex index = new BucketDelayedMessageIndex(); + DelayedMessageIndex index = new DelayedMessageIndex(); Long2ObjectMap firstBucket = new Long2ObjectOpenHashMap<>(); LongBitmap bits = LongBitmaps.create(); @@ -160,7 +160,7 @@ public void restoreIsIdempotentOnOverlappingSnapshots() { @Test public void restoreAfterTrackMergesCorrectly() { - BucketDelayedMessageIndex index = new BucketDelayedMessageIndex(); + DelayedMessageIndex index = new DelayedMessageIndex(); index.track(1L, 1L); assertThat(index.size()).isEqualTo(1L); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java index 0e3f130cbf168..10a3dd6a3f91f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/InMemoryDeliveryTrackerTest.java @@ -34,10 +34,12 @@ import java.lang.reflect.Method; import java.time.Clock; import java.util.NavigableMap; +import java.util.NavigableSet; import java.util.TreeMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import lombok.Cleanup; +import org.apache.bookkeeper.mledger.Position; import org.apache.pulsar.broker.service.persistent.AbstractPersistentDispatcherMultipleConsumers; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -259,4 +261,21 @@ public void testAddMultipleMessagesSameWindow(InMemoryDelayedDeliveryTracker tra tracker.getScheduledMessages(10); } + + @Test(dataProvider = "delayedTracker") + public void testDuplicateAddMessageIsTrackedOnce(InMemoryDelayedDeliveryTracker tracker) throws Exception { + assertTrue(tracker.addMessage(1, 5, 10)); + assertTrue(tracker.addMessage(1, 5, 10)); + assertTrue(tracker.addMessage(1, 6, 10)); + assertTrue(tracker.addMessage(2, 5, 10)); + assertEquals(tracker.getNumberOfDelayedMessages(), 3); + + clockTime.set(20); + + NavigableSet scheduled = tracker.getScheduledMessages(100); + assertEquals(scheduled.size(), 3); + assertEquals(tracker.getNumberOfDelayedMessages(), 0); + + tracker.close(); + } } From 8b87c1fea5de957cb2b7a865304c85c369996e6e Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Fri, 28 Aug 2026 21:20:29 +0800 Subject: [PATCH 203/213] [improve][ml] Per-msgLedger individual ack checkpoint persistence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Squashed from the optimize-TripleLongPriorityQueue-heap-operations branch (10 commits on top of #26127) and adapted to the current PositionRangeSet-based code. Instead of serializing the full individualDeletedMessages snapshot on every flush, the cursor ledger becomes a structured checkpoint log: - CursorCheckpointLog: CursorLogEntry envelope; oversized checkpoints are transparently chunked (CursorCheckpointChunk), recovery scans back bounded by MAX_SCAN_BACK and falls back to legacy PositionInfo bytes - CursorCheckpointPersistence: per-msgLedger incremental writes — AckState carries one bitmap per changed ledger inline, unchanged ledgers only store an AckStateRef to their previously persisted entry; self-heals active ledgers without a checkpoint - Reset is consistency-first: in-memory state applies only after the tombstone is durable; on BK failure falls back to an md-only ZK tombstone (complete because a post-reset state has no holes) - shouldPersistUnackRangesToLedger always targets BK when enabled, so holes never collapse into ZK metadata - Cursor stats: individualDeletedMessagesCount / firstIndividualDeleted Message (ManagedCursorMXBean, metrics, internal stats), cardinality guard for markDeletePosition >= lastPosition, and truncation recording - Gated by persistentUnackedRangesWithPerLedgerEntryEnabled (default false); persistentUnackedRangesMaxEntrySize defaults to 5MB. Dirty-ledger tracking is tied to this flag alone; the legacy persistentUnackedRangesWithMultipleEntriesEnabled no longer has any runtime effect. Metrics aligned with apache/master and extended: - Truncation OTel counters match apache/master exactly, with its edge-triggered warning (lastCursorDataFullyPersistable); the long-array persistence path instruments the same counter via a toRanges callback, closing a silent-truncation blind spot upstream still has - New OTel metrics: cursor.ack.operation.count and cursor.recover.operation.count observables (zero hot-path cost) plus cursor.ack/persist/recover.latency DoubleHistograms (unit s); legacy brk_ml_cursor_* Prometheus gauges kept - Attribute resolution failures (non-topic-named managed ledgers) are contained so telemetry recording never breaks callback chains Adaptations for this branch: recoverIndividualDeletedMessages keeps the List-based entry point delegating to the (int, IntFunction) accessor form; test logging kept on the fork's explicit slf4j logger; restored the 5-arg MarkDeleteEntry convenience constructor used by NonDurableCursorImpl (missing on the source branch tip). Verification: build + checkstyle green; tests pass — CursorCheckpointLogRecoveryTest 13, PositionRangeSetDirtyTrackingTest 16, PositionRangeSetTest 22, PositionRangeSetCompatibilityTest 5, ManagedCursorTest#testCheckpoint* 27, OrderingTest 1. --- conf/broker.conf | 6 + managed-ledger/pom.xml | 5 + .../bookkeeper/mledger/ManagedCursor.java | 19 + .../mledger/ManagedCursorMXBean.java | 62 + .../mledger/ManagedLedgerConfig.java | 32 + .../mledger/impl/CursorCheckpointLog.java | 416 ++++++ .../impl/CursorCheckpointPersistence.java | 487 +++++++ .../bookkeeper/mledger/impl/EntryImpl.java | 4 + .../mledger/impl/ManagedCursorImpl.java | 599 +++++++- .../mledger/impl/ManagedCursorMXBeanImpl.java | 77 + .../impl/ManagedLedgerFactoryImpl.java | 4 + .../mledger/impl/ManagedLedgerImpl.java | 3 +- .../impl/OpenTelemetryManagedCursorStats.java | 127 +- .../mledger/impl/PositionRangeSet.java | 165 ++- .../src/main/proto/MLDataFormats.proto | 39 + .../impl/CursorCheckpointLogRecoveryTest.java | 355 +++++ ...rsorCheckpointPersistenceOrderingTest.java | 70 + .../mledger/impl/ManagedCursorTest.java | 1262 +++++++++++++++++ .../PositionRangeSetCompatibilityTest.java | 6 +- .../PositionRangeSetDirtyTrackingTest.java | 193 +++ .../mledger/impl/PositionRangeSetTest.java | 61 + .../pulsar/broker/ServiceConfiguration.java | 14 +- .../pulsar/broker/service/BrokerService.java | 2 + .../stats/metrics/ManagedCursorMetrics.java | 8 + .../data/ManagedLedgerInternalStats.java | 3 + .../client/PulsarMockBookKeeper.java | 13 + 26 files changed, 3922 insertions(+), 110 deletions(-) create mode 100644 managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointLog.java create mode 100644 managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointPersistence.java create mode 100644 managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointLogRecoveryTest.java create mode 100644 managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointPersistenceOrderingTest.java create mode 100644 managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetDirtyTrackingTest.java diff --git a/conf/broker.conf b/conf/broker.conf index 667e10f11f122..609656b7b986d 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -2085,6 +2085,12 @@ dispatcherPauseOnAckStatePersistentEnabled=false # multiple entries. persistentUnackedRangesWithMultipleEntriesEnabled=false +# Enables per-msgLedger cursor checkpoint persistence. When true, the cursor writes a +# CursorCheckpoint per flush (mark-delete ledger's ack state inline + refs to other ledgers' +# previously-persisted ack states) instead of the legacy single PositionInfo entry. Eliminates +# write amplification and avoids ack truncation. +persistentUnackedRangesWithPerLedgerEntryEnabled=false + # Deprecated - Use managedLedgerCacheEvictionIntervalMs instead managedLedgerCacheEvictionFrequency=0 diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index 69dad6affc389..e2f16c5561242 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -48,6 +48,11 @@ protobuf-java + + it.unimi.dsi + fastutil + + ${project.groupId} pulsar-common diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java index 340a5c4306287..659b162a7658c 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java @@ -371,6 +371,25 @@ default void asyncReadEntriesWithSkipOrWait(int maxEntries, long maxSizeBytes, R */ long getNumberOfEntriesInBacklog(boolean isPrecise); + /** + * Return whether this cursor has non-deleted messages in backlog. + * + * @return true if there is at least one entry in backlog + */ + default boolean hasBacklog() { + return hasBacklog(true); + } + + /** + * Return whether this cursor has non-deleted messages in backlog. + * + * @param isPrecise set to true to get a precise backlog check + * @return true if there is at least one entry in backlog + */ + default boolean hasBacklog(boolean isPrecise) { + return getNumberOfEntriesInBacklog(isPrecise) > 0; + } + /** * This signals that the reader is done with all the entries up to "position" (included). This can potentially * trigger a ledger deletion, if all the other cursors are done too with the underlying ledger. diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursorMXBean.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursorMXBean.java index 7402bd65f793e..dc04fc4923429 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursorMXBean.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursorMXBean.java @@ -100,4 +100,66 @@ public interface ManagedCursorMXBean { */ long getReadCursorLedgerSize(); + /** + * Record an acknowledgment operation and its processing latency. + * + * @param latencyMillis acknowledgment processing latency in milliseconds + */ + void recordAck(long latencyMillis); + + /** + * @return the number of acknowledgment operations + */ + long getAckCount(); + + /** + * @return the average acknowledgment processing latency in milliseconds + */ + double getAckLatencyAvgMillis(); + + /** + * Record a cursor persist (checkpoint write to the cursor ledger) operation and its latency. + * + * @param latencyMillis persist latency in milliseconds + */ + void recordPersist(long latencyMillis); + + /** + * @return the number of cursor persist operations + */ + long getPersistCount(); + + /** + * @return the average cursor persist latency in milliseconds + */ + double getPersistLatencyAvgMillis(); + + /** + * Record a cursor recovery operation and its latency. + * + * @param latencyMillis recovery latency in milliseconds + * @param success whether the recovery completed successfully + */ + void recordRecover(long latencyMillis, boolean success); + + /** + * @return the number of cursor recovery operations + */ + long getRecoverCount(); + + /** + * @return the number of successful cursor recovery operations + */ + long getRecoverSucceed(); + + /** + * @return the number of failed cursor recovery operations + */ + long getRecoverErrors(); + + /** + * @return the average cursor recovery latency in milliseconds + */ + double getRecoverLatencyAvgMillis(); + } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java index 0eaebaf0f8cfb..eb3d4844d6d48 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java @@ -45,6 +45,22 @@ public class ManagedLedgerConfig { private int maxUnackedRangesToPersist = 10000; private int maxBatchDeletedIndexToPersist = 10000; private boolean persistentUnackedRangesWithMultipleEntriesEnabled = false; + /** + * Enables per-msgLedger cursor checkpoint persistence. When true, the cursor + * writes one {@code CursorCheckpoint} per flush (with the mark-delete ledger's ack state + * inline and refs to other ledgers' previously-persisted ack states) instead of the + * legacy single-{@code PositionInfo} entry. + * + *

See {@code CursorLogEntry}, {@code CursorCheckpoint}, {@code AckState}, and + * {@code AckStateRef} in {@code MLDataFormats.proto} for the wire format. + */ + private boolean persistentUnackedRangesWithPerLedgerEntryEnabled = false; + /** + * Maximum serialized size for a single {@code CursorLogEntry}. Entries exceeding + * this are transparently chunked. Not exposed as a broker config — defaults to the same + * 5 MB as {@code maxMessageSize} so chunk behavior mirrors the message-size limit. + */ + private int persistentUnackedRangesMaxEntrySize = 5 * 1024 * 1024; private boolean deletionAtBatchIndexLevelEnabled = true; private int maxUnackedRangesToPersistInMetadataStore = 1000; private int maxEntriesPerLedger = 50000; @@ -505,6 +521,22 @@ public void setPersistentUnackedRangesWithMultipleEntriesEnabled(boolean multipl this.persistentUnackedRangesWithMultipleEntriesEnabled = multipleEntriesEnabled; } + public boolean isPersistentUnackedRangesWithPerLedgerEntryEnabled() { + return persistentUnackedRangesWithPerLedgerEntryEnabled; + } + + public void setPersistentUnackedRangesWithPerLedgerEntryEnabled(boolean enabled) { + this.persistentUnackedRangesWithPerLedgerEntryEnabled = enabled; + } + + public int getPersistentUnackedRangesMaxEntrySize() { + return persistentUnackedRangesMaxEntrySize; + } + + public void setPersistentUnackedRangesMaxEntrySize(int maxEntrySize) { + this.persistentUnackedRangesMaxEntrySize = maxEntrySize; + } + /** * @param maxUnackedRangesToPersist * max unacked message ranges that will be persisted and receverd. diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointLog.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointLog.java new file mode 100644 index 0000000000000..fdaedfbcf150f --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointLog.java @@ -0,0 +1,416 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.impl; + +import com.google.protobuf.ByteString; +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.proto.MLDataFormats.CursorCheckpoint; +import org.apache.bookkeeper.mledger.proto.MLDataFormats.CursorCheckpointChunk; +import org.apache.bookkeeper.mledger.proto.MLDataFormats.CursorLogEntry; +import org.apache.bookkeeper.mledger.proto.MLDataFormats.PositionInfo; +import org.apache.pulsar.common.util.FutureUtil; + +@Slf4j +class CursorCheckpointLog { + + private static final long MAX_SCAN_BACK = 1000L; + + private final int maxEntrySize; + private final int chunkEnvelopeOverhead; + private volatile long zkCheckpointLedgerId = -1; + private volatile long zkCheckpointEntryId = -1; + + CursorCheckpointLog(int maxEntrySize) { + if (maxEntrySize < 1024) { + throw new IllegalArgumentException("maxEntrySize must be at least 1024 bytes"); + } + this.maxEntrySize = maxEntrySize; + CursorLogEntry probe = CursorLogEntry.newBuilder() + .setCheckpointChunk(CursorCheckpointChunk.newBuilder() + .setPartIndex(0).setPartCount(1) + .setCheckpointBytes(ByteString.copyFrom(new byte[0])) + .build()) + .build(); + this.chunkEnvelopeOverhead = probe.toByteArray().length + 8; + } + + void setZkCheckpointHint(long cursorLedgerId, long entryId) { + this.zkCheckpointLedgerId = cursorLedgerId; + this.zkCheckpointEntryId = entryId; + } + + CompletableFuture appendCheckpoint(LedgerHandle lh, CursorCheckpoint checkpoint) { + byte[] checkpointBytes = checkpoint.toByteArray(); + CursorLogEntry envelope; + try { + envelope = CursorLogEntry.newBuilder() + .setCheckpoint(CursorCheckpoint.parseFrom(checkpointBytes)) + .build(); + } catch (Exception e) { + return FutureUtil.failedFuture(new ManagedLedgerException("Failed to parse checkpoint", e)); + } + byte[] data = envelope.toByteArray(); + if (data.length <= maxEntrySize) { + return addEntry(lh, data).thenApply(entryId -> { + log.debug("Appended checkpoint, ledgerId: {}, entryId: {}, size: {}", + lh.getId(), entryId, data.length); + return new AppendResult(data.length, entryId); + }); + } + int maxPayloadSize = maxEntrySize - chunkEnvelopeOverhead; + int partCount = (checkpointBytes.length + maxPayloadSize - 1) / maxPayloadSize; + log.debug("Appending chunked checkpoint, ledgerId: {}, checkpointBytes: {}, partCount: {}", + lh.getId(), checkpointBytes.length, partCount); + List> futures = new ArrayList<>(partCount); + int offset = 0; + for (int i = 0; i < partCount; i++) { + int length = Math.min(maxPayloadSize, checkpointBytes.length - offset); + byte[] payload = new byte[length]; + System.arraycopy(checkpointBytes, offset, payload, 0, length); + offset += length; + CursorLogEntry part = CursorLogEntry.newBuilder() + .setCheckpointChunk(CursorCheckpointChunk.newBuilder() + .setPartIndex(i).setPartCount(partCount) + .setCheckpointBytes(ByteString.copyFrom(payload)) + .build()) + .build(); + futures.add(addEntry(lh, part.toByteArray())); + } + CompletableFuture lastPartFuture = futures.get(partCount - 1); + return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) + .thenCompose(ignored -> lastPartFuture + .thenApply(lastEntryId -> { + log.debug("Appended chunked checkpoint, ledgerId: {}, lastEntryId: {}, " + + "partCount: {}, totalBytes: {}", + lh.getId(), lastEntryId, partCount, checkpointBytes.length); + return new AppendResult(checkpointBytes.length, lastEntryId); + })); + } + + CompletableFuture readLatest(LedgerHandle lh) { + long last = lh.getLastAddConfirmed(); + if (last < 0) { + return FutureUtil.failedFuture(new ManagedLedgerException( + "Cursor ledger " + lh.getId() + " has no entries")); + } + log.debug("Recovering checkpoint, ledgerId: {}, lastEntryId: {}", lh.getId(), last); + return recoverEntryAt(lh, last, true).thenCompose(decision -> { + if (decision.shouldScanBack) { + log.info("Scanning back for last complete checkpoint, ledgerId: {}, entryId: {}", + lh.getId(), last); + return scanBack(lh, last); + } + return CompletableFuture.completedFuture(decision.state); + }); + } + + /** + * Recovers the checkpoint stored at a specific entry, assembling chunked checkpoints when + * needed. Unlike {@link #readLatest}, this never scans back: the target of an ack state ref + * must be a complete checkpoint at exactly the given entry, so anything else fails fast. + */ + CompletableFuture readAt(LedgerHandle lh, long entryId) { + return recoverEntryAt(lh, entryId, false).thenCompose(decision -> { + if (decision.shouldScanBack) { + return FutureUtil.failedFuture(new ManagedLedgerException( + "No complete checkpoint at entry " + entryId + " in ledger " + lh.getId())); + } + return CompletableFuture.completedFuture(decision.state); + }); + } + + /** + * Reads and decodes the entry at {@code entryId}. When {@code fallbackToScanBack} is set, + * decode failures yield a scan-back decision (used by the tail/scan-back paths); otherwise they + * propagate to the caller. + */ + private CompletableFuture recoverEntryAt(LedgerHandle lh, long entryId, + boolean fallbackToScanBack) { + CompletableFuture recovered = readEntry(lh, entryId) + .thenCompose(bytes -> recoverEntry(lh, entryId, bytes)); + return fallbackToScanBack + ? recovered.exceptionally(error -> { + log.warn("Failed to recover entry, ledgerId: {}, entryId: {}", lh.getId(), entryId, error); + return RecoveryDecision.scanBack(); + }) + : recovered; + } + + private CompletableFuture recoverEntry(LedgerHandle lh, long entryId, byte[] bytes) { + if (bytes == null) { + return FutureUtil.failedFuture(new ManagedLedgerException( + "Entry " + entryId + " in cursor ledger " + lh.getId() + " is empty")); + } + + CursorLogEntry envelope; + try { + envelope = CursorLogEntry.parseFrom(bytes); + } catch (Exception e) { + if (isLegacyPositionInfo(bytes)) { + // Legacy PositionInfo bytes can fail CursorLogEntry parsing. + log.debug("Recovered legacy PositionInfo after CursorLogEntry parse failure, " + + "ledgerId: {}, entryId: {}", lh.getId(), entryId); + return CompletableFuture.completedFuture(RecoveryDecision.recovered(RecoveredState.legacy(bytes))); + } + return FutureUtil.failedFuture(new ManagedLedgerException( + "Invalid cursor log entry at ledger " + lh.getId() + " entry " + entryId + + ": failed to parse CursorLogEntry", e)); + } + if (envelope.hasCheckpoint()) { + CursorCheckpoint cp = envelope.getCheckpoint(); + log.debug("Recovered checkpoint, ledgerId: {}, entryId: {}, mdLedgerId: {}, mdEntryId: {}, " + + "ackStates: {}, ackStateRefs: {}", + lh.getId(), entryId, cp.getMarkDeleteLedgerId(), cp.getMarkDeleteEntryId(), + cp.getAckStatesCount(), cp.getAckStateRefsCount()); + return CompletableFuture.completedFuture( + RecoveryDecision.recovered(new RecoveredState(cp, entryId))); + } + if (envelope.hasCheckpointChunk()) { + CursorCheckpointChunk chunk = envelope.getCheckpointChunk(); + int partCount = chunk.getPartCount(); + int partIndex = chunk.getPartIndex(); + if (partIndex >= 0 && partIndex < partCount) { + return recoverFromChunk(lh, entryId, chunk); + } + } + if (isLegacyPositionInfo(bytes)) { + // Backward compatibility: legacy PositionInfo bytes can parse as an empty CursorLogEntry. + log.debug("Recovered legacy PositionInfo from unknown CursorLogEntry payload, " + + "ledgerId: {}, entryId: {}", lh.getId(), entryId); + return CompletableFuture.completedFuture(RecoveryDecision.recovered(RecoveredState.legacy(bytes))); + } + return FutureUtil.failedFuture(new ManagedLedgerException( + "Invalid cursor log entry at ledger " + lh.getId() + " entry " + entryId + + ": neither checkpoint nor checkpointChunk")); + } + + private CompletableFuture recoverFromChunk( + LedgerHandle lh, long entryId, CursorCheckpointChunk chunk) { + int partCount = chunk.getPartCount(); + int partIndex = chunk.getPartIndex(); + if (partIndex < 0 || partIndex >= partCount) { + log.warn("Invalid chunk metadata, ledgerId: {}, entryId: {}, partIndex: {}, partCount: {}", + lh.getId(), entryId, partIndex, partCount); + return FutureUtil.failedFuture(new ManagedLedgerException( + "Invalid chunk metadata at ledger " + lh.getId() + " entry " + entryId + + ": partIndex=" + partIndex + ", partCount=" + partCount)); + } + if (partIndex != partCount - 1) { + log.debug("Chunk is not last part, scanning back, ledgerId: {}, entryId: {}, partIndex: {}, partCount: {}", + lh.getId(), entryId, partIndex, partCount); + return CompletableFuture.completedFuture(RecoveryDecision.scanBack()); + } + return assemble(lh, entryId, partCount) + .handle((cp, error) -> { + if (error != null) { + log.warn("Chunk assembly failed, scanning back, ledgerId: {}, entryId: {}, partCount: {}", + lh.getId(), entryId, partCount, error); + return RecoveryDecision.scanBack(); + } + return RecoveryDecision.recovered(new RecoveredState(cp, entryId)); + }); + } + + private CompletableFuture assemble(LedgerHandle lh, long lastPartEntryId, int partCount) { + long firstId = lastPartEntryId - partCount + 1; + if (firstId < 0) { + return FutureUtil.failedFuture(new ManagedLedgerException( + "Chunk assembly underflow: first part entry id " + firstId + + " (lastPartEntryId=" + lastPartEntryId + ", partCount=" + partCount + ")")); + } + return readEntryRange(lh, firstId, lastPartEntryId).thenApply(partsBytes -> { + if (partsBytes.size() != partCount) { + throw new RuntimeException(new ManagedLedgerException( + "Chunk assembly mismatch: expected " + partCount + ", got " + partsBytes.size())); + } + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + for (int i = 0; i < partCount; i++) { + CursorLogEntry part; + try { + part = CursorLogEntry.parseFrom(partsBytes.get(i)); + } catch (Exception e) { + throw new RuntimeException(new ManagedLedgerException("Failed to parse chunk part " + i, e)); + } + if (!part.hasCheckpointChunk()) { + throw new RuntimeException(new ManagedLedgerException("Part " + i + " is not a chunk")); + } + CursorCheckpointChunk chunk = part.getCheckpointChunk(); + if (chunk.getPartIndex() != i || chunk.getPartCount() != partCount) { + throw new RuntimeException(new ManagedLedgerException( + "Part " + i + ": partIndex=" + chunk.getPartIndex() + + " partCount=" + chunk.getPartCount() + ", expected " + i + "/" + partCount)); + } + byte[] checkpointBytes = chunk.getCheckpointBytes().toByteArray(); + bos.write(checkpointBytes, 0, checkpointBytes.length); + } + try { + CursorCheckpoint cp = CursorCheckpoint.parseFrom(bos.toByteArray()); + log.debug("Assembled chunked checkpoint, ledgerId: {}, partCount: {}, assembledBytes: {}", + lh.getId(), partCount, bos.size()); + return cp; + } catch (Exception e) { + throw new RuntimeException(new ManagedLedgerException("Failed to parse assembled checkpoint", e)); + } + }); + } + + private CompletableFuture scanBack(LedgerHandle lh, long fromEntryId) { + CompletableFuture hintFuture = null; + if (zkCheckpointLedgerId >= 0 && zkCheckpointEntryId >= 0 + && zkCheckpointLedgerId == lh.getId() && zkCheckpointEntryId < fromEntryId) { + log.debug("Trying ZK checkpoint hint before scan-back, ledgerId: {}, hintEntryId: {}, fromEntryId: {}", + lh.getId(), zkCheckpointEntryId, fromEntryId); + hintFuture = recoverEntryAt(lh, zkCheckpointEntryId, false) + .thenCompose(decision -> decision.shouldScanBack + ? scanBackStep(lh, fromEntryId - 1) + : CompletableFuture.completedFuture(decision.state)); + } + if (hintFuture != null) { + return hintFuture.exceptionally(t -> { + log.debug("ZK checkpoint hint failed, falling back to sequential scan-back, " + + "ledgerId: {}, hintEntryId: {}", + lh.getId(), zkCheckpointEntryId); + return null; + }).thenCompose(state -> + state != null ? CompletableFuture.completedFuture(state) + : scanBackStep(lh, fromEntryId - 1)); + } + return scanBackStep(lh, fromEntryId - 1); + } + + private CompletableFuture scanBackStep(LedgerHandle lh, long entryId) { + long floor = lh.getLastAddConfirmed() - MAX_SCAN_BACK; + if (entryId < 0 || entryId < floor) { + log.warn("Scan-back exhausted without recoverable checkpoint, ledgerId: {}, entryId: {}, floor: {}", + lh.getId(), entryId, floor); + return FutureUtil.failedFuture(new ManagedLedgerException( + "scanBack exhausted without finding a complete checkpoint")); + } + return recoverEntryAt(lh, entryId, true) + .thenCompose(decision -> decision.shouldScanBack + ? scanBackStep(lh, entryId - 1) + : CompletableFuture.completedFuture(decision.state)); + } + + private static CompletableFuture addEntry(LedgerHandle lh, byte[] data) { + CompletableFuture future = new CompletableFuture<>(); + lh.asyncAddEntry(data, (rc, handle, entryId, ctx) -> { + if (rc == BKException.Code.OK) { + future.complete(entryId); + } else { + future.completeExceptionally(BKException.create(rc)); + } + }, null); + return future; + } + + private static CompletableFuture readEntry(LedgerHandle lh, long entryId) { + CompletableFuture future = new CompletableFuture<>(); + lh.asyncReadEntries(entryId, entryId, (rc, lh1, entries, ctx) -> { + if (rc != BKException.Code.OK) { + future.completeExceptionally(BKException.create(rc)); + return; + } + if (!entries.hasMoreElements()) { + future.complete(null); + return; + } + future.complete(entries.nextElement().getEntry()); + }, null); + return future; + } + + private static CompletableFuture> readEntryRange(LedgerHandle lh, long firstId, long lastId) { + CompletableFuture> future = new CompletableFuture<>(); + lh.asyncReadEntries(firstId, lastId, (rc, lh1, entries, ctx) -> { + if (rc != BKException.Code.OK) { + future.completeExceptionally(BKException.create(rc)); + return; + } + List result = new ArrayList<>(); + while (entries.hasMoreElements()) { + result.add(entries.nextElement().getEntry()); + } + future.complete(result); + }, null); + return future; + } + + private static boolean isLegacyPositionInfo(byte[] bytes) { + try { + PositionInfo.parseFrom(bytes); + return true; + } catch (Exception e) { + return false; + } + } + + record AppendResult(int totalBytes, long commitEntryId) {} + + private static final class RecoveryDecision { + private final RecoveredState state; + private final boolean shouldScanBack; + + private RecoveryDecision(RecoveredState state, boolean shouldScanBack) { + this.state = state; + this.shouldScanBack = shouldScanBack; + } + + static RecoveryDecision recovered(RecoveredState state) { + return new RecoveryDecision(state, false); + } + + static RecoveryDecision scanBack() { + return new RecoveryDecision(null, true); + } + } + + static final class RecoveredState { + final CursorCheckpoint checkpoint; + final long commitEntryId; + final byte[] legacyBytes; + + private RecoveredState(CursorCheckpoint checkpoint, long commitEntryId) { + this.checkpoint = checkpoint; + this.commitEntryId = commitEntryId; + this.legacyBytes = null; + } + + private RecoveredState(byte[] legacyBytes) { + this.checkpoint = null; + this.commitEntryId = -1; + this.legacyBytes = legacyBytes; + } + + static RecoveredState legacy(byte[] bytes) { + return new RecoveredState(bytes); + } + + boolean isLegacy() { + return legacyBytes != null; + } + } +} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointPersistence.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointPersistence.java new file mode 100644 index 0000000000000..fd677d7dd3de6 --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointPersistence.java @@ -0,0 +1,487 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.impl; + +import com.google.common.annotations.VisibleForTesting; +import com.google.protobuf.ByteString; +import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.locks.ReadWriteLock; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.bookkeeper.mledger.proto.MLDataFormats.AckState; +import org.apache.bookkeeper.mledger.proto.MLDataFormats.AckStateRef; +import org.apache.bookkeeper.mledger.proto.MLDataFormats.BatchedEntryDeletionIndexInfo; +import org.apache.bookkeeper.mledger.proto.MLDataFormats.CursorCheckpoint; +import org.apache.bookkeeper.mledger.proto.MLDataFormats.LongProperty; + +@Slf4j +class CursorCheckpointPersistence { + + private final CursorCheckpointLog writer; + private final ReadWriteLock lock; + private final BookKeeper bookKeeper; + private final BookKeeper.DigestType digestType; + private final byte[] password; + @Getter + private final boolean perLedgerEntryPersistEnabled; + + // Position of each msg ledger's latest checkpoint in the cursor ledger; the AckStateRef + // targets are derived from this index. + private final Long2ObjectOpenHashMap lastCheckpointPos = new Long2ObjectOpenHashMap<>(); + private volatile CompletableFuture lastPersist = + CompletableFuture.completedFuture(null); + + public CursorCheckpointPersistence(CursorCheckpointLog writer, ReadWriteLock lock, + BookKeeper bookKeeper, + BookKeeper.DigestType digestType, byte[] password, + boolean perLedgerEntryPersistEnabled) { + this.writer = writer; + this.lock = lock; + this.bookKeeper = bookKeeper; + this.digestType = digestType; + this.password = password; + this.perLedgerEntryPersistEnabled = perLedgerEntryPersistEnabled; + } + + public void setZkCheckpointHint(long cursorLedgerId, long entryId) { + writer.setZkCheckpointHint(cursorLedgerId, entryId); + } + + public synchronized CompletableFuture persist( + LedgerHandle lh, Position mdPos, Map properties, + ManagedCursorImpl cursor) { + lastPersist = lastPersist + .exceptionally(e -> null) + .thenCompose(ignored -> doPersist(lh, mdPos, properties, cursor)); + return lastPersist; + } + + /** + * Immutable snapshot of cursor state for one flush. Building reads only from this snapshot, + * not live cursor state, so acks arriving during async appends don't wedge the flush. + */ + private static final class PersistContext { + final long mdLedgerId; + // Ledgers whose checkpoint is written by this flush (dirty + self-healed positionless + // active ledgers), excluding the mark-delete ledger. + final Set flushedLedgers; + // Ledgers that may be referenced by this flush's checkpoints. + final Set activeLedgers; + final Map dirtyBitmaps; + final Map> dirtyBatchAcks; + final byte[] mdBitmap; + final List mdBatchAcks; + + PersistContext(long mdLedgerId, Set flushedLedgers, Set activeLedgers, + Map dirtyBitmaps, + Map> dirtyBatchAcks, + byte[] mdBitmap, List mdBatchAcks) { + this.mdLedgerId = mdLedgerId; + this.flushedLedgers = flushedLedgers; + this.activeLedgers = activeLedgers; + this.dirtyBitmaps = dirtyBitmaps; + this.dirtyBatchAcks = dirtyBatchAcks; + this.mdBitmap = mdBitmap; + this.mdBatchAcks = mdBatchAcks; + } + } + + private PersistContext createPersistContext(ManagedCursorImpl cursor, Position mdPos) { + lock.readLock().lock(); + try { + long mdLedgerId = mdPos.getLedgerId(); + Set clearedDirtyLedgers = cursor.individualDeletedMessages.snapshotAndClearDirtyLedgers(); + Set flushedLedgers = new HashSet<>(clearedDirtyLedgers); + flushedLedgers.remove(mdLedgerId); + + Set activeLedgers = new HashSet<>(); + cursor.individualDeletedMessages.forEachActiveLedger(activeLedgers::add); + Map> batchAcksByLedger = groupBatchAcksByLedger(cursor); + batchAcksByLedger.keySet().forEach(activeLedgers::add); + + // Self-heal: active ledgers without a persisted checkpoint are flushed this round, + // otherwise they'd have no position to reference. + for (long id : activeLedgers) { + if (id != mdLedgerId && !lastCheckpointPos.containsKey(id)) { + flushedLedgers.add(id); + } + } + + Map dirtyBitmaps = new HashMap<>(); + Map> dirtyBatchAcks = new HashMap<>(); + for (long id : flushedLedgers) { + dirtyBitmaps.put(id, cursor.individualDeletedMessages.bitmapOf(id)); + dirtyBatchAcks.put(id, batchAcksByLedger.getOrDefault(id, Collections.emptyList())); + } + byte[] mdBitmap = cursor.individualDeletedMessages.bitmapOf(mdLedgerId); + List mdBatchAcks = + batchAcksByLedger.getOrDefault(mdLedgerId, Collections.emptyList()); + return new PersistContext(mdLedgerId, flushedLedgers, activeLedgers, dirtyBitmaps, + dirtyBatchAcks, mdBitmap, mdBatchAcks); + } finally { + lock.readLock().unlock(); + } + } + + private static Map> groupBatchAcksByLedger( + ManagedCursorImpl cursor) { + Map> result = new HashMap<>(); + if (cursor.batchDeletedIndexes == null) { + return result; + } + cursor.batchDeletedIndexes.forEach((position, bitSet) -> { + List list = + result.computeIfAbsent(position.getLedgerId(), k -> new ArrayList<>()); + BatchedEntryDeletionIndexInfo.Builder infoBuilder = BatchedEntryDeletionIndexInfo.newBuilder(); + infoBuilder.getPositionBuilder() + .setLedgerId(position.getLedgerId()) + .setEntryId(position.getEntryId()); + bitSet.stream().forEach(infoBuilder::addDeleteSet); + list.add(infoBuilder.build()); + }); + return result; + } + + private CompletableFuture doPersist( + LedgerHandle lh, Position mdPos, Map properties, + ManagedCursorImpl cursor) { + PersistContext ctx = createPersistContext(cursor, mdPos); + + // Drop positions for ledgers below mark-delete that are no longer active. Active ledgers + // are kept: batch-index entries may still be in memory before the align cleanup runs. + lock.writeLock().lock(); + try { + lastCheckpointPos.keySet().removeIf(id -> id < ctx.mdLedgerId && !ctx.activeLedgers.contains(id)); + } finally { + lock.writeLock().unlock(); + } + + List dirtyOrder = new ArrayList<>(ctx.flushedLedgers); + Collections.sort(dirtyOrder); + // mdLedger is written first so other checkpoints in this flush can reference its position. + List writeOrder = new ArrayList<>(dirtyOrder.size() + 1); + writeOrder.add(ctx.mdLedgerId); + writeOrder.addAll(dirtyOrder); + + final Set appendedLedgers = new HashSet<>(); + CompletableFuture chain = + CompletableFuture.completedFuture(null); + for (long ledgerId : writeOrder) { + chain = chain.thenCompose(ignored -> { + byte[] bitmap = ledgerId == ctx.mdLedgerId ? ctx.mdBitmap : ctx.dirtyBitmaps.get(ledgerId); + List batchAcks = + ledgerId == ctx.mdLedgerId ? ctx.mdBatchAcks : ctx.dirtyBatchAcks.get(ledgerId); + CursorCheckpoint checkpoint = buildCheckpoint(ctx, ledgerId, bitmap, batchAcks, + mdPos, properties); + return writer.appendCheckpoint(lh, checkpoint).thenApply(result -> { + lock.writeLock().lock(); + try { + Position persistedPos = PositionFactory.create(lh.getId(), result.commitEntryId()); + recordCheckpointPos(ledgerId, persistedPos); + appendedLedgers.add(ledgerId); + } finally { + lock.writeLock().unlock(); + } + return result; + }); + }); + } + + return chain.exceptionally(error -> { + restoreDirtyForFailedLedgers(cursor, ctx.flushedLedgers, appendedLedgers); + throw new CompletionException(error); + }); + } + + /** + * Re-marks dirty for ledgers whose checkpoint was not appended, so the next flush retries them. + */ + /** + * Records the position of a msg ledger's latest checkpoint. Persists are serialized + * ({@link #persist} chains behind the previous one), so puts normally arrive in order; + * the monotonic guard is defense-in-depth: a stale completion must never regress the + * index, otherwise a later AckStateRef would point at an older (smaller) ack bitmap + * and the acks recorded in the newer checkpoint would be lost on recovery. + */ + @VisibleForTesting + void recordCheckpointPos(long msgLedgerId, Position pos) { + Position prev = lastCheckpointPos.get(msgLedgerId); + if (prev == null || pos.compareTo(prev) > 0) { + lastCheckpointPos.put(msgLedgerId, pos); + } + } + + @VisibleForTesting + Position checkpointPosOf(long msgLedgerId) { + return lastCheckpointPos.get(msgLedgerId); + } + + private void restoreDirtyForFailedLedgers(ManagedCursorImpl cursor, Set flushedLedgers, + Set appendedLedgers) { + Set failedLedgers = new HashSet<>(flushedLedgers); + failedLedgers.removeAll(appendedLedgers); + if (failedLedgers.isEmpty()) { + return; + } + lock.writeLock().lock(); + try { + cursor.individualDeletedMessages.restoreDirtyLedgers(failedLedgers); + } finally { + lock.writeLock().unlock(); + } + } + + // ============================ recover ============================ + + public CompletableFuture recover(LedgerHandle lh) { + return writer.readLatest(lh).thenCompose(state -> { + if (state.isLegacy()) { + return CompletableFuture.completedFuture(RecoveredCheckpoint.legacy(state.legacyBytes)); + } + CursorCheckpoint cp = state.checkpoint; + return fetchAckStateRefs(cp, lh).thenApply(fetched -> { + validateRecoveredAckData(cp, fetched); + rebuildLastCheckpointPos(lh, state.commitEntryId, cp); + return RecoveredCheckpoint.of(cp, fetched); + }); + }); + } + + /** Validates no duplicate msgLedgerIds and that all refs resolved. Fails fast on corruption. */ + private static void validateRecoveredAckData(CursorCheckpoint cp, Map fetched) { + Set expected = new HashSet<>(cp.getAckStatesCount() + cp.getAckStateRefsCount()); + for (AckState ackState : cp.getAckStatesList()) { + if (!expected.add(ackState.getMsgLedgerId())) { + throw new RuntimeException(new ManagedLedgerException( + "Recovery inconsistency: duplicate msgLedgerId " + ackState.getMsgLedgerId() + + " in inline ack states")); + } + } + for (AckStateRef ref : cp.getAckStateRefsList()) { + if (!expected.add(ref.getMsgLedgerId())) { + throw new RuntimeException(new ManagedLedgerException( + "Recovery inconsistency: msgLedgerId " + ref.getMsgLedgerId() + + " appears both inline and in ack state refs")); + } + } + if (!fetched.keySet().equals(expected)) { + throw new RuntimeException(new ManagedLedgerException( + "Recovery inconsistency: expected ack states for " + expected + + ", got " + fetched.keySet())); + } + } + + /** Rebuilds lastCheckpointPos from the recovered checkpoint so the first persist can emit refs. */ + private void rebuildLastCheckpointPos(LedgerHandle lh, long commitEntryId, CursorCheckpoint cp) { + lock.writeLock().lock(); + try { + lastCheckpointPos.clear(); + Position inlinePos = PositionFactory.create(lh.getId(), commitEntryId); + for (AckState ackState : cp.getAckStatesList()) { + lastCheckpointPos.put(ackState.getMsgLedgerId(), inlinePos); + } + for (AckStateRef ref : cp.getAckStateRefsList()) { + lastCheckpointPos.put(ref.getMsgLedgerId(), + PositionFactory.create(ref.getCursorLedgerId(), ref.getEntryId())); + } + } finally { + lock.writeLock().unlock(); + } + } + + public CompletableFuture recoverWithHint( + LedgerHandle lh, long hintCursorLedgerId, long hintEntryId) { + if (hintCursorLedgerId >= 0 && hintEntryId >= 0) { + setZkCheckpointHint(hintCursorLedgerId, hintEntryId); + } + return recover(lh); + } + + private CompletableFuture> fetchAckStateRefs(CursorCheckpoint cp, LedgerHandle lh) { + Map result = new HashMap<>(); + for (AckState ackState : cp.getAckStatesList()) { + result.put(ackState.getMsgLedgerId(), toAckStateData(ackState)); + } + List refs = cp.getAckStateRefsList(); + if (refs.isEmpty()) { + return CompletableFuture.completedFuture(result); + } + List> fetches = new ArrayList<>(refs.size()); + for (AckStateRef ref : refs) { + fetches.add(fetchAckState(ref, lh)); + } + return CompletableFuture.allOf(fetches.toArray(new CompletableFuture[0])) + .thenApply(ignored -> { + for (int i = 0; i < refs.size(); i++) { + result.put(refs.get(i).getMsgLedgerId(), fetches.get(i).join()); + } + return result; + }); + } + + private CompletableFuture fetchAckState(AckStateRef ref, LedgerHandle lh) { + return readCheckpointFromLedger(lh, ref.getCursorLedgerId(), ref.getEntryId()) + .thenApply(cp -> extractAckState(cp, ref)); + } + + private static AckStateData extractAckState(CursorCheckpoint cp, AckStateRef ref) { + for (AckState ackState : cp.getAckStatesList()) { + if (ackState.getMsgLedgerId() == ref.getMsgLedgerId()) { + return toAckStateData(ackState); + } + } + throw new RuntimeException(new ManagedLedgerException( + "AckStateRef target " + ref + " does not contain msgLedgerId " + + ref.getMsgLedgerId())); + } + + private static AckStateData toAckStateData(AckState ackState) { + return new AckStateData( + ackState.hasAckBitmap() ? ackState.getAckBitmap().toByteArray() : null, + ackState.getBatchAcksList() != null + ? ackState.getBatchAcksList() : Collections.emptyList()); + } + + private CompletableFuture readCheckpointFromLedger( + LedgerHandle lh, long ledgerId, long entryId) { + CompletableFuture future = new CompletableFuture<>(); + if (ledgerId == lh.getId()) { + // Refs into the recovered cursor ledger itself can reuse the already-open handle. + writer.readAt(lh, entryId) + .whenComplete((state, error) -> completeCheckpointRead(future, ledgerId, entryId, state, error)); + return future; + } + bookKeeper.asyncOpenLedgerNoRecovery(ledgerId, digestType, password, (rc, handle, ctx) -> { + if (rc != BKException.Code.OK) { + future.completeExceptionally(BKException.create(rc)); + return; + } + writer.readAt(handle, entryId).whenComplete((state, error) -> { + handle.asyncClose((closeRc, closeHandle, closeCtx) -> {}, null); + completeCheckpointRead(future, ledgerId, entryId, state, error); + }); + }, null); + return future; + } + + private static void completeCheckpointRead(CompletableFuture future, long ledgerId, + long entryId, CursorCheckpointLog.RecoveredState state, + Throwable error) { + if (error != null) { + future.completeExceptionally(error); + } else if (state.isLegacy()) { + future.completeExceptionally(new ManagedLedgerException( + "AckStateRef target entry " + entryId + " in ledger " + ledgerId + + " is not a checkpoint")); + } else { + future.complete(state.checkpoint); + } + } + + // ============================ checkpoint builder ============================ + + /** + * Builds a checkpoint for one msg ledger: inline ack state plus refs to every other active + * ledger's latest position. Refs stay valid because cursor ledgers are only GC'd once + * mark-delete passes the last ledger holding acks. + */ + private CursorCheckpoint buildCheckpoint(PersistContext ctx, long msgLedgerId, byte[] bitmap, + List batchAcks, + Position mdPos, Map properties) { + CursorCheckpoint.Builder cpBuilder = CursorCheckpoint.newBuilder() + .setMarkDeleteLedgerId(mdPos.getLedgerId()) + .setMarkDeleteEntryId(mdPos.getEntryId()); + if (properties != null) { + properties.forEach((name, value) -> { + cpBuilder.addProperties(LongProperty.newBuilder().setName(name).setValue(value).build()); + }); + } + addAckState(cpBuilder, msgLedgerId, bitmap, batchAcks); + lock.readLock().lock(); + try { + for (long id : ctx.activeLedgers) { + if (id != msgLedgerId) { + Position pos = lastCheckpointPos.get(id); + if (pos == null) { + if (ctx.flushedLedgers.contains(id)) { + continue; + } + throw new IllegalStateException( + "Missing lastCheckpointPos for active msgLedgerId " + id); + } + cpBuilder.addAckStateRefs(AckStateRef.newBuilder() + .setMsgLedgerId(id) + .setCursorLedgerId(pos.getLedgerId()) + .setEntryId(pos.getEntryId()) + .build()); + } + } + } finally { + lock.readLock().unlock(); + } + return cpBuilder.build(); + } + + private static void addAckState(CursorCheckpoint.Builder cp, long msgLedgerId, byte[] bitmap, + List batchAcks) { + AckState.Builder ackStateBuilder = AckState.newBuilder().setMsgLedgerId(msgLedgerId); + if (bitmap != null && bitmap.length > 0) { + ackStateBuilder.setAckBitmap(ByteString.copyFrom(bitmap)); + } + if (batchAcks != null && !batchAcks.isEmpty()) { + ackStateBuilder.addAllBatchAcks(batchAcks); + } + cp.addAckStates(ackStateBuilder.build()); + } + + // ============================ DTOs ============================ + + record AckStateData(byte[] ackBitmap, List batchAcks) {} + + record RecoveredCheckpoint(CursorCheckpoint checkpoint, + Map ackData, + byte[] legacyBytes) { + static RecoveredCheckpoint of(CursorCheckpoint cp, Map ackData) { + return new RecoveredCheckpoint(cp, ackData, null); + } + + static RecoveredCheckpoint legacy(byte[] bytes) { + return new RecoveredCheckpoint(null, null, bytes); + } + + boolean isLegacy() { + return legacyBytes != null; + } + } +} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryImpl.java index e0e2b859794b5..87a037a90f60d 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryImpl.java @@ -94,6 +94,10 @@ public static EntryImpl create(Position position, ByteBuf data) { return entry; } + public static EntryImpl create(Position position, ByteBuf data, int expectedReadCount) { + return create(position, data); + } + public static EntryImpl create(EntryImpl other) { EntryImpl entry = RECYCLER.get(); entry.timestamp = System.nanoTime(); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 9820f32f85149..44e3ffb508b02 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -33,11 +33,11 @@ import com.google.common.collect.Lists; import com.google.common.collect.Range; import com.google.common.util.concurrent.RateLimiter; -import com.google.protobuf.InvalidProtocolBufferException; import java.time.Clock; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.BitSet; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -48,6 +48,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -96,6 +97,9 @@ import org.apache.bookkeeper.mledger.ScanOutcome; import org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback; import org.apache.bookkeeper.mledger.proto.MLDataFormats; +import org.apache.bookkeeper.mledger.proto.MLDataFormats.AckStateRef; +import org.apache.bookkeeper.mledger.proto.MLDataFormats.BatchedEntryDeletionIndexInfo; +import org.apache.bookkeeper.mledger.proto.MLDataFormats.CursorCheckpoint; import org.apache.bookkeeper.mledger.proto.MLDataFormats.LongListMap; import org.apache.bookkeeper.mledger.proto.MLDataFormats.LongProperty; import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo; @@ -105,6 +109,7 @@ import org.apache.bookkeeper.mledger.proto.MLDataFormats.PositionInfo.Builder; import org.apache.bookkeeper.mledger.proto.MLDataFormats.StringProperty; import org.apache.bookkeeper.mledger.util.ManagedLedgerUtils; +import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.commons.lang3.mutable.MutableInt; import org.apache.commons.lang3.mutable.MutableLong; import org.apache.commons.lang3.tuple.Pair; @@ -208,6 +213,11 @@ public class ManagedCursorImpl implements ManagedCursor { @Nullable protected final ConcurrentSkipListMap batchDeletedIndexes; protected final ReadWriteLock lock = new ReentrantReadWriteLock(); + @VisibleForTesting + final CursorCheckpointPersistence ackPersistence; + private volatile long pendingCheckpointHintLedgerId = -1; + private volatile long pendingCheckpointHintEntryId = -1; + private final Set allCursorLedgerIds = ConcurrentHashMap.newKeySet(); private RateLimiter markDeleteLimiter; // The cursor is considered "dirty" when there are mark-delete updates that are only applied in memory, // because of the rate limiting. @@ -220,6 +230,7 @@ public class ManagedCursorImpl implements ManagedCursor { private long entriesReadCount; private long entriesReadSize; private int individualDeletedMessagesSerializedSize; + private final AtomicBoolean lastCursorDataFullyPersistable = new AtomicBoolean(true); private static final String COMPACTION_CURSOR_NAME = "__compaction"; private volatile boolean cacheReadEntry = false; @@ -237,6 +248,13 @@ class MarkDeleteEntry { final Object ctx; final Map properties; final Runnable alignAcknowledgeStatusAfterPersisted; + // True for custom aligns (e.g. resetCursor) that must run even when persist is deferred. + // Entries that carry a custom align runnable (cursor reset) must surface persist + // failures to their caller instead of silently reporting success: the in-memory + // state was NOT applied, so a "successful" reset would be a silent no-op. + final boolean propagatePersistFailure; + // Set once the position is actually durable; gates the default align. + volatile boolean persistedSuccessfully; // If the callbackGroup is set, it means this mark-delete request was done on behalf of a group of request (just // persist the last one in the chain). In this case we need to trigger the callbacks for every request in the @@ -245,11 +263,17 @@ class MarkDeleteEntry { public MarkDeleteEntry(Position newPosition, Map properties, MarkDeleteCallback callback, Object ctx) { - this(newPosition, properties, callback, ctx, null); + this(newPosition, properties, callback, ctx, null, false); } public MarkDeleteEntry(Position newPosition, Map properties, MarkDeleteCallback callback, Object ctx, Runnable alignAcknowledgeStatusAfterPersisted) { + this(newPosition, properties, callback, ctx, alignAcknowledgeStatusAfterPersisted, false); + } + + public MarkDeleteEntry(Position newPosition, Map properties, + MarkDeleteCallback callback, Object ctx, Runnable alignAcknowledgeStatusAfterPersisted, + boolean propagatePersistFailure) { if (alignAcknowledgeStatusAfterPersisted == null) { alignAcknowledgeStatusAfterPersisted = () -> { if (batchDeletedIndexes != null) { @@ -265,6 +289,7 @@ public MarkDeleteEntry(Position newPosition, Map properties, this.callback = callback; this.ctx = ctx; this.alignAcknowledgeStatusAfterPersisted = alignAcknowledgeStatusAfterPersisted; + this.propagatePersistFailure = propagatePersistFailure; } public void triggerComplete() { @@ -363,13 +388,18 @@ protected ManagedCursorImpl(BookKeeper bookkeeper, ManagedLedgerImpl ledger, Str this.ledger = ledger; this.name = cursorName; this.individualDeletedMessages = new PositionRangeSet(positionRangeConverter, - getConfig().isPersistentUnackedRangesWithMultipleEntriesEnabled()); + getConfig().isPersistentUnackedRangesWithPerLedgerEntryEnabled()); if (getConfig().isDeletionAtBatchIndexLevelEnabled()) { this.batchDeletedIndexes = new ConcurrentSkipListMap<>(); } else { this.batchDeletedIndexes = null; } this.digestType = BookKeeper.DigestType.fromApiDigestType(getConfig().getDigestType()); + CursorCheckpointLog writer = new CursorCheckpointLog(getConfig().getPersistentUnackedRangesMaxEntrySize()); + this.ackPersistence = new CursorCheckpointPersistence( + writer, lock, bookkeeper, + this.digestType, getConfig().getPassword(), + getConfig().isPersistentUnackedRangesWithPerLedgerEntryEnabled()); PENDING_MARK_DELETED_SUBMITTED_COUNT_UPDATER.set(this, 0); PENDING_READ_OPS_UPDATER.set(this, 0); RESET_CURSOR_IN_PROGRESS_UPDATER.set(this, FALSE); @@ -398,6 +428,24 @@ public Map getProperties() { return lastMarkDeleteEntry != null ? lastMarkDeleteEntry.properties : Collections.emptyMap(); } + private void recordAckStats(long latencyMillis) { + mbean.recordAck(latencyMillis); + ledger.getFactory().getOpenTelemetryManagedCursorStats() + .recordAckLatency(this, latencyMillis / 1000.0); + } + + private void recordPersistStats(long latencyMillis) { + mbean.recordPersist(latencyMillis); + ledger.getFactory().getOpenTelemetryManagedCursorStats() + .recordPersistLatency(this, latencyMillis / 1000.0); + } + + private void recordRecoverStats(long latencyMillis, boolean success) { + mbean.recordRecover(latencyMillis, success); + ledger.getFactory().getOpenTelemetryManagedCursorStats() + .recordRecoverLatency(this, latencyMillis / 1000.0); + } + @Override public boolean isCursorDataFullyPersistable() { lock.readLock().lock(); @@ -543,6 +591,22 @@ public boolean removeProperty(String key) { void recover(final VoidCallback callback) { // Read the meta-data ledgerId from the store log.info("[{}] Recovering from bookkeeper ledger cursor: {}", ledger.getName(), name); + final long recoverStartTimeNanos = System.nanoTime(); + final VoidCallback recoverCallback = new VoidCallback() { + @Override + public void operationComplete() { + recordRecoverStats( + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - recoverStartTimeNanos), true); + callback.operationComplete(); + } + + @Override + public void operationFailed(ManagedLedgerException exception) { + recordRecoverStats( + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - recoverStartTimeNanos), false); + callback.operationFailed(exception); + } + }; ledger.getStore().asyncGetCursorInfo(ledger.getName(), name, new MetaStoreCallback() { @Override public void operationComplete(ManagedCursorInfo info, Stat stat) { @@ -583,18 +647,18 @@ public void operationComplete(ManagedCursorInfo info, Stat stat) { } recoveredCursor(recoveredPosition, recoveredProperties, recoveredCursorProperties, null); - callback.operationComplete(); + recoverCallback.operationComplete(); } else { // Need to proceed and read the last entry in the specified ledger to find out the last position log.info("[{}] Cursor {} meta-data recover from ledger {}", ledger.getName(), name, info.getCursorsLedgerId()); - recoverFromLedger(info, callback); + recoverFromLedger(info, recoverCallback); } } @Override public void operationFailed(MetaStoreException e) { - callback.operationFailed(e); + recoverCallback.operationFailed(e); } }); } @@ -652,32 +716,7 @@ protected void recoverFromLedger(final ManagedCursorInfo info, final VoidCallbac LedgerEntry entry = seq.nextElement(); mbean.addReadCursorLedgerSize(entry.getLength()); - PositionInfo positionInfo; - try { - positionInfo = PositionInfo.parseFrom(entry.getEntry()); - } catch (InvalidProtocolBufferException e) { - callback.operationFailed(new ManagedLedgerException(e)); - return; - } - - Map recoveredProperties = Collections.emptyMap(); - if (positionInfo.getPropertiesCount() > 0) { - // Recover properties map - recoveredProperties = new HashMap<>(); - for (int i = 0; i < positionInfo.getPropertiesCount(); i++) { - LongProperty property = positionInfo.getProperties(i); - recoveredProperties.put(property.getName(), property.getValue()); - } - } - - Position position = PositionFactory.create(positionInfo.getLedgerId(), positionInfo.getEntryId()); - recoverIndividualDeletedMessages(positionInfo); - if (getConfig().isDeletionAtBatchIndexLevelEnabled() - && positionInfo.getBatchedEntryDeletionIndexInfoCount() > 0) { - recoverBatchDeletedIndexes(positionInfo.getBatchedEntryDeletionIndexInfoList()); - } - recoveredCursor(position, recoveredProperties, cursorProperties, lh); - callback.operationComplete(); + recoverWithCheckpointFormat(info, lh, ledgerId, entry.getEntry(), callback); }, null); }; try { @@ -690,6 +729,27 @@ protected void recoverFromLedger(final ManagedCursorInfo info, final VoidCallbac } } + private void recoverWithCheckpointFormat(ManagedCursorInfo info, LedgerHandle lh, long cursorLedgerId, + byte[] lastEntryBytes, VoidCallback callback) { + long hintLedgerId = info.hasLastCmCursorLedgerId() ? info.getLastCmCursorLedgerId() : -1; + long hintEntryId = info.hasLastCmCursorLedgerId() ? info.getLastCmEntryId() : -1; + ackPersistence.recoverWithHint(lh, hintLedgerId, hintEntryId).whenComplete((recovered, recoverError) -> { + if (recoverError != null) { + Throwable cause = FutureUtil.unwrapCompletionException(recoverError); + log.warn("Per-msgLedger checkpoint recovery failed, rewinding to ZK snapshot, ledgerId: {}", + cursorLedgerId, cause); + initialize(getRollbackPosition(info), Collections.emptyMap(), cursorProperties, callback); + return; + } + if (recovered != null && !recovered.isLegacy()) { + applyRecoveredCheckpoint(recovered, info, lh, cursorProperties, callback); + return; + } + byte[] legacyBytes = recovered != null ? recovered.legacyBytes() : lastEntryBytes; + applyLegacyPositionInfo(legacyBytes, info, lh, cursorProperties, callback); + }); + } + public void recoverIndividualDeletedMessages(PositionInfo positionInfo) { if (positionInfo.getIndividualDeletedMessagesCount() > 0) { recoverIndividualDeletedMessages(positionInfo.getIndividualDeletedMessagesList()); @@ -733,10 +793,16 @@ private List buildLongPropertiesMap(Map properties) { } private void recoverIndividualDeletedMessages(List individualDeletedMessagesList) { + recoverIndividualDeletedMessages(individualDeletedMessagesList.size(), individualDeletedMessagesList::get); + } + + @VisibleForTesting + void recoverIndividualDeletedMessages(int count, IntFunction accessor) { lock.writeLock().lock(); try { individualDeletedMessages.clear(); - individualDeletedMessagesList.forEach(messageRange -> { + for (int i = 0; i < count; i++) { + MessageRange messageRange = accessor.apply(i); MLDataFormats.NestedPositionInfo lowerEndpoint = messageRange.getLowerEndpoint(); MLDataFormats.NestedPositionInfo upperEndpoint = messageRange.getUpperEndpoint(); @@ -763,21 +829,12 @@ private void recoverIndividualDeletedMessages(List i individualDeletedMessages.addOpenClosed(upperEndpoint.getLedgerId(), -1, upperEndpoint.getLedgerId(), upperEndpoint.getEntryId()); } - }); + } } finally { lock.writeLock().unlock(); } } - @VisibleForTesting - void recoverIndividualDeletedMessages(int count, IntFunction accessor) { - List individualDeletedMessagesList = new ArrayList<>(count); - for (int i = 0; i < count; i++) { - individualDeletedMessagesList.add(accessor.apply(i)); - } - recoverIndividualDeletedMessages(individualDeletedMessagesList); - } - private void recoverBatchDeletedIndexes ( List batchDeletedIndexInfoList) { Objects.requireNonNull(batchDeletedIndexes); @@ -800,6 +857,100 @@ private void recoverBatchDeletedIndexes ( } } + /** + * Applies a recovered {@link CursorCheckpointPersistence.RecoveredCheckpoint} to + * the cursor's in-memory state, then delegates to {@link #recoveredCursor} for finalization. + */ + private void applyRecoveredCheckpoint(CursorCheckpointPersistence.RecoveredCheckpoint recovered, + ManagedCursorInfo info, LedgerHandle lh, + Map cursorProperties, + VoidCallback callback) { + CursorCheckpoint cp = recovered.checkpoint(); + Map recoveredProperties = Collections.emptyMap(); + if (cp.getPropertiesCount() > 0) { + recoveredProperties = new HashMap<>(); + for (int i = 0; i < cp.getPropertiesCount(); i++) { + LongProperty prop = cp.getProperties(i); + recoveredProperties.put(prop.getName(), prop.getValue()); + } + } + Position position = PositionFactory.create(cp.getMarkDeleteLedgerId(), cp.getMarkDeleteEntryId()); + + // Track the recovered cursor ledger and every referenced old cursor ledger so GC can + // reclaim them across restarts once mark-delete passes the last ledger holding acks. + allCursorLedgerIds.add(lh.getId()); + for (AckStateRef ref : cp.getAckStateRefsList()) { + allCursorLedgerIds.add(ref.getCursorLedgerId()); + } + + // Apply per-ledger ack bitmaps via bulk build (RoaringBitmap bytes → LongBitmap). + Map bitmaps = new HashMap<>(); + for (Map.Entry e : recovered.ackData().entrySet()) { + byte[] bm = e.getValue().ackBitmap(); + if (bm != null && bm.length > 0) { + bitmaps.put(e.getKey(), bm); + } + } + lock.writeLock().lock(); + try { + individualDeletedMessages.buildFromBitmaps(bitmaps); + // Apply batch acks (across all ledgers) + if (getConfig().isDeletionAtBatchIndexLevelEnabled() && batchDeletedIndexes != null) { + for (CursorCheckpointPersistence.AckStateData data : recovered.ackData().values()) { + if (data.batchAcks() != null) { + for (BatchedEntryDeletionIndexInfo batchInfo : data.batchAcks()) { + Position batchPos = PositionFactory.create( + batchInfo.getPosition().getLedgerId(), batchInfo.getPosition().getEntryId()); + BitSet bitSet = new BitSet(); + for (int i = 0; i < batchInfo.getDeleteSetCount(); i++) { + bitSet.set((int) batchInfo.getDeleteSet(i)); + } + batchDeletedIndexes.put(batchPos, bitSet); + } + } + } + } + // Dirty state is fresh after recovery — nothing to persist until next ack. + individualDeletedMessages.resetDirtyKeys(); + } finally { + lock.writeLock().unlock(); + } + + recoveredCursor(position, recoveredProperties, cursorProperties, lh); + callback.operationComplete(); + } + + /** + * Legacy fallback: parses bytes as {@link PositionInfo} and runs the existing + * recovery path. + */ + private void applyLegacyPositionInfo(byte[] bytes, ManagedCursorInfo info, LedgerHandle lh, + Map cursorProperties, VoidCallback callback) { + PositionInfo positionInfo; + try { + positionInfo = PositionInfo.parseFrom(bytes); + } catch (Exception e) { + callback.operationFailed(new ManagedLedgerException(e)); + return; + } + Map recoveredProperties = Collections.emptyMap(); + if (positionInfo.getPropertiesCount() > 0) { + recoveredProperties = new HashMap<>(); + for (int i = 0; i < positionInfo.getPropertiesCount(); i++) { + LongProperty property = positionInfo.getProperties(i); + recoveredProperties.put(property.getName(), property.getValue()); + } + } + Position position = PositionFactory.create(positionInfo.getLedgerId(), positionInfo.getEntryId()); + recoverIndividualDeletedMessages(positionInfo); + if (getConfig().isDeletionAtBatchIndexLevelEnabled() + && positionInfo.getBatchedEntryDeletionIndexInfoCount() > 0) { + recoverBatchDeletedIndexes(positionInfo.getBatchedEntryDeletionIndexInfoList()); + } + recoveredCursor(position, recoveredProperties, cursorProperties, lh); + callback.operationComplete(); + } + private void recoveredCursor(Position position, Map properties, Map cursorProperties, LedgerHandle recoveredFromCursorLedger) { @@ -1280,6 +1431,24 @@ public int getNonContiguousDeletedMessagesRangeSerializedSize() { return this.individualDeletedMessagesSerializedSize; } + long getNumberOfIndividualDeletedMessages() { + return individualDeletedMessages.totalCardinality(); + } + + String getFirstIndividualDeletedMessage() { + lock.readLock().lock(); + try { + Range first = individualDeletedMessages.firstRange(); + if (first == null) { + return null; + } + return PositionFactory.create(first.lowerEndpoint().getLedgerId(), + first.lowerEndpoint().getEntryId() + 1).toString(); + } finally { + lock.readLock().unlock(); + } + } + @Override public long getEstimatedSizeSinceMarkDeletePosition() { Position markDeletePosition = this.markDeletePosition; @@ -1381,6 +1550,50 @@ public long getNumberOfEntriesInBacklog(boolean isPrecise) { return backlog; } + @Override + public boolean hasBacklog() { + Position markDeletePosition = this.markDeletePosition; + Position lastPosition = ledger.getLastPosition(); + if (markDeletePosition == null || markDeletePosition.compareTo(lastPosition) >= 0) { + return false; + } + + Position nextPosition = ledger.getNextValidPosition(markDeletePosition); + if (nextPosition.compareTo(lastPosition) > 0) { + return false; + } + + lock.readLock().lock(); + try { + while (nextPosition.compareTo(lastPosition) <= 0) { + Range deletedRange = individualDeletedMessages.rangeContaining( + nextPosition.getLedgerId(), nextPosition.getEntryId()); + if (deletedRange == null) { + return true; + } + + Position upperEndpoint = deletedRange.upperEndpoint(); + if (upperEndpoint.compareTo(lastPosition) >= 0) { + return false; + } + nextPosition = ledger.getNextValidPosition(upperEndpoint); + } + return false; + } finally { + lock.readLock().unlock(); + } + } + + @Override + public boolean hasBacklog(boolean isPrecise) { + if (isPrecise) { + return hasBacklog(); + } + + long backlog = ManagedLedgerImpl.ENTRIES_ADDED_COUNTER_UPDATER.get(ledger) - messagesConsumedCounter; + return backlog >= 0 ? backlog > 0 : hasBacklog(); + } + public long getNumberOfEntriesInStorage() { return ledger.getNumberOfEntries(Range.openClosed(markDeletePosition, ledger.getLastPosition())); } @@ -1605,6 +1818,8 @@ protected void internalResetCursor(Position proposedReadPosition, MSG_CONSUMED_COUNTER_UPDATER.addAndGet(ManagedCursorImpl.this, -ackedEntriesAfterMdPosition.get().longValue()); markDeletePosition = newMarkDeletePosition; + // Clear: a racing mark-delete may have advanced this past the reset position. + persistentMarkDeletePosition = null; lastMarkDeleteEntry = new MarkDeleteEntry(newMarkDeletePosition, isCompactionCursor() ? getProperties() : Collections.emptyMap(), null, null); individualDeletedMessages.clear(); @@ -1614,6 +1829,7 @@ protected void internalResetCursor(Position proposedReadPosition, long[] resetWords = ackSetState.getAckSet(); if (resetWords != null) { batchDeletedIndexes.put(newReadPosition, BitSet.valueOf(resetWords)); + individualDeletedMessages.markDirtyLedger(newReadPosition.getLedgerId()); } }); } @@ -2204,9 +2420,23 @@ public MarkDeletingMarkedPosition(String s) { public void asyncMarkDelete(final Position position, Map properties, final MarkDeleteCallback callback, final Object ctx) { requireNonNull(position); + final long ackStartTimeNanos = System.nanoTime(); + final MarkDeleteCallback ackCallback = new MarkDeleteCallback() { + @Override + public void markDeleteComplete(Object ctx) { + recordAckStats(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - ackStartTimeNanos)); + callback.markDeleteComplete(ctx); + } + + @Override + public void markDeleteFailed(ManagedLedgerException exception, Object ctx) { + recordAckStats(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - ackStartTimeNanos)); + callback.markDeleteFailed(exception, ctx); + } + }; if (isClosed()) { - callback.markDeleteFailed(new ManagedLedgerException + ackCallback.markDeleteFailed(new ManagedLedgerException .CursorAlreadyClosedException("Cursor was already closed"), ctx); return; } @@ -2216,7 +2446,7 @@ public void asyncMarkDelete(final Position position, Map propertie log.debug("[{}] cursor reset in progress - ignoring mark delete on position [{}] for cursor [{}]", ledger.getName(), position, name); } - callback.markDeleteFailed( + ackCallback.markDeleteFailed( new ManagedLedgerException("Reset cursor in progress - unable to mark delete position " + position.toString()), ctx); @@ -2250,7 +2480,7 @@ public void asyncMarkDelete(final Position position, Map propertie log.debug("[{}] Failed mark delete due to invalid markDelete {} is ahead of last-confirmed-entry {}" + " for cursor [{}]", ledger.getName(), position, lastConfirmedEntry, name); } - callback.markDeleteFailed(new ManagedLedgerException("Invalid mark deleted position"), ctx); + ackCallback.markDeleteFailed(new ManagedLedgerException("Invalid mark deleted position"), ctx); return; } } @@ -2259,7 +2489,7 @@ public void asyncMarkDelete(final Position position, Map propertie try { newPosition = setAcknowledgedPosition(newPosition); } catch (IllegalArgumentException e) { - callback.markDeleteFailed(getManagedLedgerException(e), ctx); + ackCallback.markDeleteFailed(getManagedLedgerException(e), ctx); return; } finally { lock.writeLock().unlock(); @@ -2269,10 +2499,10 @@ public void asyncMarkDelete(final Position position, Map propertie if (markDeleteLimiter != null && !markDeleteLimiter.tryAcquire()) { isDirty = true; updateLastMarkDeleteEntryToLatest(newPosition, properties); - callback.markDeleteComplete(ctx); + ackCallback.markDeleteComplete(ctx); return; } - internalAsyncMarkDelete(newPosition, properties, callback, ctx, null); + internalAsyncMarkDelete(newPosition, properties, ackCallback, ctx, null); } private Position ackBatchPosition(Position position) { @@ -2311,7 +2541,7 @@ protected void internalAsyncMarkDelete(final Position newPosition, Map propertiesToUse = properties != null ? properties : (last != null ? last.properties : getProperties()); MarkDeleteEntry mdEntry = new MarkDeleteEntry(newPosition, propertiesToUse, callback, ctx, - alignAcknowledgeStatusAfterPersisted); + alignAcknowledgeStatusAfterPersisted, alignAcknowledgeStatusAfterPersisted != null); // The state might have changed while we were waiting on the queue mutex switch (state) { @@ -2408,7 +2638,9 @@ public void operationComplete() { // point. lock.writeLock().lock(); try { - mdEntry.alignAcknowledgeStatus(); + if (mdEntry.persistedSuccessfully) { + mdEntry.alignAcknowledgeStatus(); + } } finally { lock.writeLock().unlock(); } @@ -2512,8 +2744,38 @@ public void deleteFailed(ManagedLedgerException exception, Object ctx) { @Override public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallback callback, Object ctx) { + Iterable ackPositions = positions; + if (!(ackPositions instanceof Collection)) { + List materialized = new ArrayList<>(); + for (Position position : ackPositions) { + materialized.add(position); + } + ackPositions = materialized; + } + final int ackPositionCount = ((Collection) ackPositions).size(); + final long ackStartTimeNanos = System.nanoTime(); + final AsyncCallbacks.DeleteCallback ackCallback = new AsyncCallbacks.DeleteCallback() { + @Override + public void deleteComplete(Object ctx) { + long latencyMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - ackStartTimeNanos); + for (int i = 0; i < ackPositionCount; i++) { + recordAckStats(latencyMillis); + } + callback.deleteComplete(ctx); + } + + @Override + public void deleteFailed(ManagedLedgerException exception, Object ctx) { + long latencyMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - ackStartTimeNanos); + for (int i = 0; i < ackPositionCount; i++) { + recordAckStats(latencyMillis); + } + callback.deleteFailed(exception, ctx); + } + }; + if (isClosed()) { - callback.deleteFailed(new ManagedLedgerException + ackCallback.deleteFailed(new ManagedLedgerException .CursorAlreadyClosedException("Cursor was already closed"), ctx); return; } @@ -2528,7 +2790,7 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb ledger.getName(), name, positions, individualDeletedMessages, markDeletePosition); } - for (Position pos : positions) { + for (Position pos : ackPositions) { Position position = requireNonNull(pos); if (ledger.getLastConfirmedEntry().compareTo(position) < 0) { if (log.isDebugEnabled()) { @@ -2536,7 +2798,7 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb "[{}] Failed mark delete due to invalid markDelete {} is ahead of last-confirmed-entry {} " + "for cursor [{}]", ledger.getName(), position, ledger.getLastConfirmedEntry(), name); } - callback.deleteFailed(new ManagedLedgerException("Invalid mark deleted position"), ctx); + ackCallback.deleteFailed(new ManagedLedgerException("Invalid mark deleted position"), ctx); return; } @@ -2585,6 +2847,10 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb position.getLedgerId(), position.getEntryId()); MSG_CONSUMED_COUNTER_UPDATER.incrementAndGet(this); batchDeletedIndexes.remove(position); + } else { + // Batch-index deletions are not reflected in the individual range bitmap, so + // mark the ledger dirty explicitly to ensure they are persisted. + individualDeletedMessages.markDirtyLedger(position.getLedgerId()); } } } @@ -2633,12 +2899,12 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb } catch (Exception e) { log.warn("[{}] [{}] Error while updating individualDeletedMessages [{}]", ledger.getName(), name, e.getMessage(), e); - callback.deleteFailed(getManagedLedgerException(e), ctx); + ackCallback.deleteFailed(getManagedLedgerException(e), ctx); return; } finally { lock.writeLock().unlock(); if (skipMarkDeleteBecauseAckedNothing) { - callback.deleteComplete(ctx); + ackCallback.deleteComplete(ctx); } } @@ -2646,7 +2912,7 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb if (markDeleteLimiter != null && !markDeleteLimiter.tryAcquire()) { isDirty = true; updateLastMarkDeleteEntryToLatest(newMarkDeletePosition, null); - callback.deleteComplete(ctx); + ackCallback.deleteComplete(ctx); return; } @@ -2654,12 +2920,12 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb internalAsyncMarkDelete(newMarkDeletePosition, null, new MarkDeleteCallback() { @Override public void markDeleteComplete(Object ctx) { - callback.deleteComplete(ctx); + ackCallback.deleteComplete(ctx); } @Override public void markDeleteFailed(ManagedLedgerException exception, Object ctx) { - callback.deleteFailed(exception, ctx); + ackCallback.deleteFailed(exception, ctx); } }, ctx, null); @@ -2670,7 +2936,7 @@ public void markDeleteFailed(ManagedLedgerException exception, Object ctx) { log.debug("[{}] Consumer {} cursor asyncDelete error, counters: consumed {} mdPos {} rdPos {}", ledger.getName(), name, messagesConsumedCounter, markDeletePosition, readPosition); } - callback.deleteFailed(new ManagedLedgerException(e), ctx); + ackCallback.deleteFailed(new ManagedLedgerException(e), ctx); } } @@ -2916,6 +3182,11 @@ public void operationFailed(MetaStoreException e) { private boolean shouldPersistUnackRangesToLedger() { lock.readLock().lock(); try { + // With the per-msgLedger checkpoint log enabled, closing always persists to BK: + // holes must never be collapsed into the ZK metadata (the legacy small-state path). + if (getConfig().isPersistentUnackedRangesWithPerLedgerEntryEnabled()) { + return cursorLedger != null && !isCursorLedgerReadOnly; + } return cursorLedger != null && !isCursorLedgerReadOnly && getConfig().getMaxUnackedRangesToPersist() > 0 @@ -2943,6 +3214,11 @@ private void persistPositionMetaStore(long cursorsLedgerId, Position position, M .setMarkDeleteEntryId(position.getEntryId()) // .setLastActive(lastActive); // + if (pendingCheckpointHintLedgerId >= 0) { + info.setLastCmCursorLedgerId(pendingCheckpointHintLedgerId); + info.setLastCmEntryId(pendingCheckpointHintEntryId); + } + info.addAllProperties(buildPropertiesMap(properties)); info.addAllCursorProperties(buildStringPropertiesMap(cursorProperties)); if (persistIndividualDeletedMessageRanges) { @@ -3348,8 +3624,15 @@ private List buildIndividualDeletedMessageRanges() { AtomicInteger acksSerializedSize = new AtomicInteger(0); List rangeList = new ArrayList<>(); + final int maxRanges = getConfig().getMaxUnackedRangesToPersist(); + final MutableBoolean truncated = new MutableBoolean(false); individualDeletedMessages.forEachRawRange((lowerKey, lowerValue, upperKey, upperValue) -> { + if (rangeList.size() >= maxRanges) { + truncated.setTrue(); + return false; + } + MLDataFormats.NestedPositionInfo lowerPosition = nestedPositionBuilder .setLedgerId(lowerKey) .setEntryId(lowerValue) @@ -3368,11 +3651,29 @@ private List buildIndividualDeletedMessageRanges() { acksSerializedSize.addAndGet(messageRange.getSerializedSize()); rangeList.add(messageRange); - return rangeList.size() <= getConfig().getMaxUnackedRangesToPersist(); + return true; }); this.individualDeletedMessagesSerializedSize = acksSerializedSize.get(); individualDeletedMessages.resetDirtyKeys(); + + if (truncated.booleanValue()) { + ledger.getFactory().getOpenTelemetryManagedCursorStats() + .incrementPersistUnackedRangesTruncated(this); + if (lastCursorDataFullyPersistable.compareAndSet(true, false)) { + int totalRanges = individualDeletedMessages.size(); + log.warn("[{}]-{} Individually deleted message ranges exceed" + + " managedLedgerMaxUnackedRangesToPersist (totalRanges: {}, maxRanges: {}," + + " truncated: {}). Acknowledged messages beyond this limit are not persisted" + + " and will be replayed on broker restart. Consider raising" + + " managedLedgerMaxUnackedRangesToPersist or enabling" + + " managedLedgerPersistIndividualAckAsLongArray to reduce the persisted size.", + ledger.getName(), name, totalRanges, maxRanges, totalRanges - rangeList.size()); + } + } else { + lastCursorDataFullyPersistable.compareAndSet(false, true); + } + return rangeList; } finally { lock.writeLock().unlock(); @@ -3391,7 +3692,8 @@ private List buildBatchEntryDeletio .BatchedEntryDeletionIndexInfo.newBuilder(); List result = new ArrayList<>(); final var iterator = batchDeletedIndexes.entrySet().iterator(); - while (iterator.hasNext() && result.size() < getConfig().getMaxBatchDeletedIndexToPersist()) { + final int maxIndexes = getConfig().getMaxBatchDeletedIndexToPersist(); + while (iterator.hasNext() && result.size() < maxIndexes) { final var entry = iterator.next(); nestedPositionBuilder.setLedgerId(entry.getKey().getLedgerId()); nestedPositionBuilder.setEntryId(entry.getKey().getEntryId()); @@ -3405,6 +3707,12 @@ private List buildBatchEntryDeletio batchDeletedIndexInfoBuilder.addAllDeleteSet(deleteSet); result.add(batchDeletedIndexInfoBuilder.build()); } + + if (iterator.hasNext()) { + ledger.getFactory().getOpenTelemetryManagedCursorStats() + .incrementPersistBatchDeletedIndexesTruncated(this); + } + return result; } finally { lock.readLock().unlock(); @@ -3413,6 +3721,72 @@ private List buildBatchEntryDeletio void persistPositionToLedger(final LedgerHandle lh, MarkDeleteEntry mdEntry, final VoidCallback callback, boolean ignoreClosedStateAfterFailure) { + final long persistStartTimeNanos = System.nanoTime(); + final VoidCallback persistCallback = new VoidCallback() { + @Override + public void operationComplete() { + recordPersistStats(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - persistStartTimeNanos)); + callback.operationComplete(); + } + + @Override + public void operationFailed(ManagedLedgerException exception) { + recordPersistStats(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - persistStartTimeNanos)); + callback.operationFailed(exception); + } + }; + if (ackPersistence.isPerLedgerEntryPersistEnabled()) { + ackPersistence.persist(lh, mdEntry.newPosition, mdEntry.properties, this) + .thenAccept(result -> { + mdEntry.persistedSuccessfully = true; + pendingCheckpointHintLedgerId = lh.getId(); + pendingCheckpointHintEntryId = result.commitEntryId(); + mbean.persistToLedger(true); + mbean.addWriteCursorLedgerSize(result.totalBytes()); + rolloverLedgerIfNeeded(lh); + persistCallback.operationComplete(); + }) + .exceptionally(error -> { + Throwable cause = FutureUtil.unwrapCompletionException(error); + log.warn("Per-msgLedger checkpoint persist failed, keeping state in memory;" + + " retried on the next mark-delete, ledgerId: {}, errorMessage: {}", + lh.getId(), cause.getMessage(), cause); + mbean.persistToLedger(false); + if (cause instanceof BKException) { + // BKException fences the handle; next mark-delete creates a fresh ledger. + STATE_UPDATER.compareAndSet(ManagedCursorImpl.this, State.Open, State.NoLedger); + } + // Regular mark-deletes stay tolerated and retried (at-least-once): + // ZK must never carry the large ack bitmaps. Cursor resets are different: + // their target state has no holes, so an md-only tombstone in ZK is a + // complete representation. Fall back to ZK; only if that fails too does + // the reset surface as an error (memory untouched, consistent). + if (mdEntry.propagatePersistFailure) { + persistPositionMetaStore(-1, mdEntry.newPosition, mdEntry.properties, + new MetaStoreCallback() { + @Override + public void operationComplete(Void result, Stat stat) { + // The tombstone is durable (in ZK): mark the entry so + // the align gate applies the in-memory reset. + mdEntry.persistedSuccessfully = true; + persistCallback.operationComplete(); + } + + @Override + public void operationFailed(MetaStoreException metaStoreException) { + metaStoreException.addSuppressed(cause); + persistCallback.operationFailed(createManagedLedgerException( + metaStoreException)); + } + }, false); + } else { + persistCallback.operationComplete(); + } + return null; + }); + return; + } + Position position = mdEntry.newPosition; Builder piBuilder = PositionInfo.newBuilder().setLedgerId(position.getLedgerId()) .setEntryId(position.getEntryId()) @@ -3420,10 +3794,36 @@ void persistPositionToLedger(final LedgerHandle lh, MarkDeleteEntry mdEntry, fin .addAllProperties(buildPropertiesMap(mdEntry.properties)); Map internalRanges = null; + /** + * Cursor will create the {@link #individualDeletedMessages} typed {@link LongPairRangeSet.DefaultRangeSet} if + * disabled the config {@link ManagedLedgerConfig#unackedRangesOpenCacheSetEnabled}. + * {@link LongPairRangeSet.DefaultRangeSet} never implemented the methods below: + * - {@link LongPairRangeSet#toRanges(int)}, which is used to serialize cursor metadata. + * - {@link LongPairRangeSet#build(Map)}, which is used to deserialize cursor metadata. + * Do not enable the feature that https://github.com/apache/pulsar/pull/9292 introduced, to avoid serialization + * and deserialization error. + */ + if (getConfig().isPersistIndividualAckAsLongArray()) { lock.readLock().lock(); try { - internalRanges = individualDeletedMessages.toRanges(getConfig().getMaxUnackedRangesToPersist()); + MutableBoolean truncatedLedger = new MutableBoolean(false); + internalRanges = individualDeletedMessages.toRanges(getConfig().getMaxUnackedRangesToPersist(), + skippedLedgers -> { + truncatedLedger.setTrue(); + ledger.getFactory().getOpenTelemetryManagedCursorStats() + .incrementPersistUnackedRangesTruncated(this); + if (lastCursorDataFullyPersistable.compareAndSet(true, false)) { + log.warn("[{}]-{} Truncated individual acks of {} ledgers at persistence:" + + " cumulative cardinality exceeds maxUnackedRangesToPersist={}." + + " Ack state in skipped ledgers is lost on broker restart.", + ledger.getName(), name, skippedLedgers, + getConfig().getMaxUnackedRangesToPersist()); + } + }); + if (!truncatedLedger.get()) { + lastCursorDataFullyPersistable.compareAndSet(false, true); + } } catch (Exception e) { log.warn("[{}]-{} Failed to serialize individualDeletedMessages", ledger.getName(), name, e); } finally { @@ -3451,16 +3851,17 @@ void persistPositionToLedger(final LedgerHandle lh, MarkDeleteEntry mdEntry, fin lh1.getId()); } + mdEntry.persistedSuccessfully = true; rolloverLedgerIfNeeded(lh1); mbean.persistToLedger(true); mbean.addWriteCursorLedgerSize(data.length); - callback.operationComplete(); + persistCallback.operationComplete(); } else { if (!ignoreClosedStateAfterFailure && state.isClosed()) { // After closed the cursor, the in-progress persistence task will get a // BKException.Code.LedgerClosedException. - callback.operationFailed(new CursorAlreadyClosedException(String.format("%s %s skipped this" + persistCallback.operationFailed(new CursorAlreadyClosedException(String.format("%s %s skipped this" + " persistence, because the cursor already closed", ledger.getName(), name))); return; } @@ -3471,7 +3872,7 @@ void persistPositionToLedger(final LedgerHandle lh, MarkDeleteEntry mdEntry, fin STATE_UPDATER.compareAndSet(ManagedCursorImpl.this, State.Open, State.NoLedger); // Before giving up, try to persist the position in the metadata store. - persistPositionToMetaStore(mdEntry, callback); + persistPositionToMetaStore(mdEntry, persistCallback); } }, null); } @@ -3517,6 +3918,7 @@ public void operationComplete(Void result, Stat stat) { "[{}][{}] Updated cursor in meta store after previous failure in ledger at position" + " {}", ledger.getName(), name, newPosition); } + mdEntry.persistedSuccessfully = true; mbean.persistToZookeeper(true); callback.operationComplete(); } @@ -3546,6 +3948,61 @@ boolean shouldCloseLedger(LedgerHandle lh) { } } + /** + * GC for old cursor ledgers. Uses the simplified {@code maxHold} model — when + * mark-delete has advanced past the highest msgLedgerId that still holds individual or batch + * acks, + * all referenced AckStateRefs are stale and every old cursor ledger can be deleted. + * + *

This is an all-or-nothing GC: it either deletes all tracked old ledgers or none. + * Unreferenced ledgers that were never tracked (e.g. from before a restart) leak — + * acceptable because they hold no live data. + */ + @VisibleForTesting + void gcOldCursorLedgers() { + if (!ackPersistence.isPerLedgerEntryPersistEnabled() || allCursorLedgerIds.isEmpty()) { + return; + } + long mdLedgerId = markDeletePosition.getLedgerId(); + // Find the highest msgLedgerId that currently has individual acks above mark-delete. + // individualDeletedMessages.rangeBitmapMap is not thread-safe, so the iteration must + // hold the cursor readLock to prevent concurrent mark-delete / delete modifications. + long[] maxHold = {-1}; + lock.readLock().lock(); + try { + individualDeletedMessages.forEachActiveLedger(id -> { + if (id > maxHold[0]) { + maxHold[0] = id; + } + }); + if (batchDeletedIndexes != null) { + batchDeletedIndexes.keySet().forEach(pos -> { + long id = pos.getLedgerId(); + if (id > maxHold[0]) { + maxHold[0] = id; + } + }); + } + } finally { + lock.readLock().unlock(); + } + if (maxHold[0] >= 0 && mdLedgerId < maxHold[0]) { + return; + } + for (long id : new ArrayList<>(allCursorLedgerIds)) { + log.debug("GC old cursor ledger - maxHold cleared, ledgerId: {}", id); + bookkeeper.asyncDeleteLedger(id, (rc, ctx) -> { + if (rc == BKException.Code.OK || rc == BKException.Code.NoSuchLedgerExistsException) { + allCursorLedgerIds.remove(id); + } else { + log.warn("Failed to GC old cursor ledger, will retry on next rollover, ledgerId: {}, " + + "errorMessage: {}", + id, BKException.getMessage(rc)); + } + }, null); + } + } + void switchToNewLedger(final LedgerHandle lh, final VoidCallback callback) { if (log.isDebugEnabled()) { log.debug("[{}] Switching cursor {} to ledger {}", ledger.getName(), name, lh.getId()); @@ -3563,7 +4020,12 @@ public void operationComplete(Void result, Stat stat) { // At this point the position had already been safely markdeleted callback.operationComplete(); - asyncDeleteLedger(oldLedger); + if (ackPersistence.isPerLedgerEntryPersistEnabled() && oldLedger != null) { + allCursorLedgerIds.add(oldLedger.getId()); + gcOldCursorLedgers(); + } else { + asyncDeleteLedger(oldLedger); + } } @Override @@ -4118,7 +4580,8 @@ public ManagedLedgerInternalStats.CursorStats getCursorStats() { cs.messagesConsumedCounter = getMessagesConsumedCounter(); cs.cursorLedger = getCursorLedger(); cs.cursorLedgerLastEntry = getCursorLedgerLastEntry(); - cs.individuallyDeletedMessages = getIndividuallyDeletedMessages(); + cs.individualDeletedMessagesCount = getNumberOfIndividualDeletedMessages(); + cs.firstIndividualDeletedMessage = getFirstIndividualDeletedMessage(); cs.lastLedgerSwitchTimestamp = DateFormatter.format(getLastLedgerSwitchTimestamp()); cs.state = getState(); cs.active = isActive(); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorMXBeanImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorMXBeanImpl.java index a183c0d61ce16..69b374870aeeb 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorMXBeanImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorMXBeanImpl.java @@ -34,6 +34,17 @@ public class ManagedCursorMXBeanImpl implements ManagedCursorMXBean { private final LongAdder writeCursorLedgerLogicalSize = new LongAdder(); private final LongAdder readCursorLedgerSize = new LongAdder(); + private final LongAdder ackCount = new LongAdder(); + private final LongAdder ackLatencyTotalMillis = new LongAdder(); + + private final LongAdder persistCount = new LongAdder(); + private final LongAdder persistLatencyTotalMillis = new LongAdder(); + + private final LongAdder recoverCount = new LongAdder(); + private final LongAdder recoverSucceed = new LongAdder(); + private final LongAdder recoverErrors = new LongAdder(); + private final LongAdder recoverLatencyTotalMillis = new LongAdder(); + private final ManagedCursor managedCursor; public ManagedCursorMXBeanImpl(ManagedCursor managedCursor) { @@ -114,4 +125,70 @@ public long getWriteCursorLedgerLogicalSize() { public long getReadCursorLedgerSize() { return readCursorLedgerSize.longValue(); } + + @Override + public void recordAck(long latencyMillis) { + ackCount.increment(); + ackLatencyTotalMillis.add(latencyMillis); + } + + @Override + public long getAckCount() { + return ackCount.longValue(); + } + + @Override + public double getAckLatencyAvgMillis() { + long count = ackCount.longValue(); + return count == 0 ? 0 : ackLatencyTotalMillis.longValue() / (double) count; + } + + @Override + public void recordPersist(long latencyMillis) { + persistCount.increment(); + persistLatencyTotalMillis.add(latencyMillis); + } + + @Override + public long getPersistCount() { + return persistCount.longValue(); + } + + @Override + public double getPersistLatencyAvgMillis() { + long count = persistCount.longValue(); + return count == 0 ? 0 : persistLatencyTotalMillis.longValue() / (double) count; + } + + @Override + public void recordRecover(long latencyMillis, boolean success) { + recoverCount.increment(); + recoverLatencyTotalMillis.add(latencyMillis); + if (success) { + recoverSucceed.increment(); + } else { + recoverErrors.increment(); + } + } + + @Override + public long getRecoverCount() { + return recoverCount.longValue(); + } + + @Override + public long getRecoverSucceed() { + return recoverSucceed.longValue(); + } + + @Override + public long getRecoverErrors() { + return recoverErrors.longValue(); + } + + @Override + public double getRecoverLatencyAvgMillis() { + long count = recoverCount.longValue(); + return count == 0 ? 0 : recoverLatencyTotalMillis.longValue() / (double) count; + } } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java index d45692df78fb6..8690653ec2477 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java @@ -250,6 +250,10 @@ private ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore, openTelemetryManagedCursorStats = new OpenTelemetryManagedCursorStats(openTelemetry, this); } + public OpenTelemetryManagedCursorStats getOpenTelemetryManagedCursorStats() { + return openTelemetryManagedCursorStats; + } + static class DefaultBkFactory implements BookkeeperFactoryForCustomEnsemblePlacementPolicy, AutoCloseable { private final BookKeeper bkClient; diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index a14ce1944d755..8697a7ba8ab7e 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -5023,7 +5023,8 @@ public CompletableFuture getManagedLedgerInternalSta cs.messagesConsumedCounter = cursor.getMessagesConsumedCounter(); cs.cursorLedger = cursor.getCursorLedger(); cs.cursorLedgerLastEntry = cursor.getCursorLedgerLastEntry(); - cs.individuallyDeletedMessages = cursor.getIndividuallyDeletedMessages(); + cs.individualDeletedMessagesCount = cursor.getNumberOfIndividualDeletedMessages(); + cs.firstIndividualDeletedMessage = cursor.getFirstIndividualDeletedMessage(); cs.lastLedgerSwitchTimestamp = DateFormatter.format(cursor.getLastLedgerSwitchTimestamp()); cs.state = cursor.getState(); cs.active = cursor.isActive(); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpenTelemetryManagedCursorStats.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpenTelemetryManagedCursorStats.java index ec73c9d5e5eb2..ffaed66ca20dd 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpenTelemetryManagedCursorStats.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpenTelemetryManagedCursorStats.java @@ -20,7 +20,10 @@ import com.google.common.collect.Streams; import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.metrics.BatchCallback; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.LongCounter; import io.opentelemetry.api.metrics.ObservableLongMeasurement; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedger; @@ -55,6 +58,40 @@ public class OpenTelemetryManagedCursorStats implements AutoCloseable { public static final String INCOMING_BYTE_COUNTER = "pulsar.broker.managed_ledger.cursor.incoming.size"; private final ObservableLongMeasurement incomingByteCounter; + // Broker-level counters incremented when cursor persistence silently truncates ack state. + // See managedLedgerMaxUnackedRangesToPersist and managedLedgerMaxBatchDeletedIndexToPersist. + public static final String PERSIST_UNACKED_RANGES_TRUNCATED = + "pulsar.broker.managed_ledger.cursor.persist.unacked_ranges.truncated"; + private final LongCounter persistUnackedRangesTruncated; + + public static final String PERSIST_BATCH_DELETED_INDEXES_TRUNCATED = + "pulsar.broker.managed_ledger.cursor.persist.batch_deleted_indexes.truncated"; + private final LongCounter persistBatchDeletedIndexesTruncated; + + // Replaces ['brk_ml_cursor_ackCount'] + public static final String ACK_OPERATION_COUNTER = + "pulsar.broker.managed_ledger.cursor.ack.operation.count"; + private final ObservableLongMeasurement ackOperationCounter; + + // Replaces ['brk_ml_cursor_ackLatencyAvgMs'] + public static final String ACK_LATENCY = "pulsar.broker.managed_ledger.cursor.ack.latency"; + private final DoubleHistogram ackLatency; + + // Replaces ['brk_ml_cursor_persistLatencyAvgMs']; the count is already covered by + // PERSIST_OPERATION_COUNTER. + public static final String PERSIST_LATENCY = "pulsar.broker.managed_ledger.cursor.persist.latency"; + private final DoubleHistogram persistLatency; + + // Replaces ['brk_ml_cursor_recoverCount', 'brk_ml_cursor_recoverSucceed', + // 'brk_ml_cursor_recoverErrors']. + public static final String RECOVER_OPERATION_COUNTER = + "pulsar.broker.managed_ledger.cursor.recover.operation.count"; + private final ObservableLongMeasurement recoverOperationCounter; + + // Replaces ['brk_ml_cursor_recoverLatencyAvgMs'] + public static final String RECOVER_LATENCY = "pulsar.broker.managed_ledger.cursor.recover.latency"; + private final DoubleHistogram recoverLatency; + private final BatchCallback batchCallback; public OpenTelemetryManagedCursorStats(OpenTelemetry openTelemetry, ManagedLedgerFactoryImpl factory) { @@ -96,6 +133,49 @@ public OpenTelemetryManagedCursorStats(OpenTelemetry openTelemetry, ManagedLedge .setDescription("The total amount of data read from the ledger.") .buildObserver(); + persistUnackedRangesTruncated = meter + .counterBuilder(PERSIST_UNACKED_RANGES_TRUNCATED) + .setUnit("{truncation}") + .setDescription("The number of times a cursor exceeded" + + " managedLedgerMaxUnackedRangesToPersist, causing ack state to be truncated" + + " at persistence. Ack state beyond the limit is lost on broker restart.") + .build(); + + persistBatchDeletedIndexesTruncated = meter + .counterBuilder(PERSIST_BATCH_DELETED_INDEXES_TRUNCATED) + .setUnit("{truncation}") + .setDescription("The number of times a cursor exceeded" + + " managedLedgerMaxBatchDeletedIndexToPersist, causing batch deleted index state" + + " to be truncated at persistence. State beyond the limit is lost on broker restart.") + .build(); + + ackOperationCounter = meter + .counterBuilder(ACK_OPERATION_COUNTER) + .setUnit("{operation}") + .setDescription("The number of cursor acknowledgment operations.") + .buildObserver(); + + ackLatency = meter.histogramBuilder(ACK_LATENCY) + .setUnit("s") + .setDescription("Cursor acknowledgment operation latency.") + .build(); + + persistLatency = meter.histogramBuilder(PERSIST_LATENCY) + .setUnit("s") + .setDescription("Cursor persist latency: the checkpoint write to the cursor ledger.") + .build(); + + recoverOperationCounter = meter + .counterBuilder(RECOVER_OPERATION_COUNTER) + .setUnit("{operation}") + .setDescription("The number of cursor recovery operations.") + .buildObserver(); + + recoverLatency = meter.histogramBuilder(RECOVER_LATENCY) + .setUnit("s") + .setDescription("Cursor recovery latency.") + .build(); + batchCallback = meter.batchCallback(() -> factory.getManagedLedgers() .values() .stream() @@ -107,7 +187,9 @@ public OpenTelemetryManagedCursorStats(OpenTelemetry openTelemetry, ManagedLedge nonContiguousMessageRangeCounter, outgoingByteCounter, outgoingByteLogicalCounter, - incomingByteCounter); + incomingByteCounter, + ackOperationCounter, + recoverOperationCounter); } @Override @@ -115,6 +197,14 @@ public void close() { batchCallback.close(); } + public void incrementPersistUnackedRangesTruncated(ManagedCursor cursor) { + persistUnackedRangesTruncated.add(1, cursor.getManagedCursorAttributes().getAttributes()); + } + + public void incrementPersistBatchDeletedIndexesTruncated(ManagedCursor cursor) { + persistBatchDeletedIndexesTruncated.add(1, cursor.getManagedCursorAttributes().getAttributes()); + } + private void recordMetrics(ManagedCursor cursor) { var stats = cursor.getStats(); var cursorAttributesSet = cursor.getManagedCursorAttributes(); @@ -133,5 +223,40 @@ private void recordMetrics(ManagedCursor cursor) { outgoingByteCounter.record(stats.getWriteCursorLedgerSize(), attributes); outgoingByteLogicalCounter.record(stats.getWriteCursorLedgerLogicalSize(), attributes); incomingByteCounter.record(stats.getReadCursorLedgerSize(), attributes); + + ackOperationCounter.record(stats.getAckCount(), attributes); + recoverOperationCounter.record(stats.getRecoverSucceed(), attributesSucceed); + recoverOperationCounter.record(stats.getRecoverErrors(), attributesFailed); + } + + public void recordAckLatency(ManagedCursor cursor, double seconds) { + var attributes = attributesOrNull(cursor); + if (attributes != null) { + ackLatency.record(seconds, attributes); + } + } + + public void recordPersistLatency(ManagedCursor cursor, double seconds) { + var attributes = attributesOrNull(cursor); + if (attributes != null) { + persistLatency.record(seconds, attributes); + } + } + + public void recordRecoverLatency(ManagedCursor cursor, double seconds) { + var attributes = attributesOrNull(cursor); + if (attributes != null) { + recoverLatency.record(seconds, attributes); + } + } + + private static Attributes attributesOrNull(ManagedCursor cursor) { + try { + return cursor.getManagedCursorAttributes().getAttributes(); + } catch (Exception e) { + // Attribute resolution throws for non-topic-named managed ledgers (e.g. system + // ledgers); recording telemetry must never break the caller's callback chain. + return null; + } } } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionRangeSet.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionRangeSet.java index 01e581eb444a1..82fcbf14264a4 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionRangeSet.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionRangeSet.java @@ -21,15 +21,19 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.BoundType; import com.google.common.collect.Range; -import io.github.merlimat.slog.Logger; +import io.netty.buffer.Unpooled; import it.unimi.dsi.fastutil.longs.Long2ObjectRBTreeMap; import it.unimi.dsi.fastutil.longs.Long2ObjectSortedMap; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.LongConsumer; +import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.commons.lang3.mutable.MutableInt; @@ -64,10 +68,9 @@ * because {@code managedLedgerMaxEntriesPerLedger} is an {@code int}, so valid entry ids are within * {@code [0, Integer.MAX_VALUE]}. */ +@Slf4j class PositionRangeSet implements LongPairRangeSet { - private static final Logger log = Logger.get(PositionRangeSet.class); - private static final long EARLIEST_LEDGER_ID = -1L; private static final long EARLIEST_ENTRY_ID = -1L; private static final long LATEST_LEDGER_ID = Long.MAX_VALUE; @@ -87,6 +90,7 @@ class PositionRangeSet implements LongPairRangeSet { private String cachedToString = "[]"; private boolean updatedAfterCachedForSize = true; private boolean updatedAfterCachedForToString = true; + private long totalCardinality = 0; PositionRangeSet(LongPairConsumer consumer, boolean enableMultiEntry) { this.consumer = consumer; @@ -112,21 +116,40 @@ public void addOpenClosed(long lowerLedgerId, long lowerEntryIdOpen, long upperL if (rangeBitmap != null) { long lastEntryId = rangeBitmap.lastPresentValue(); if (lastEntryId > lowerEntryIdOpen) { - rangeBitmap.add(lowerEntryId, Math.max(lastEntryId, lowerEntryId) + 1); + addRange(rangeBitmap, lowerEntryId, Math.max(lastEntryId, lowerEntryId) + 1); } } } if (isValid(upperLedgerId, upperEntryId)) { LongBitmap rangeBitmap = rangeBitmapMap.computeIfAbsent(upperLedgerId, k -> LongBitmaps.create()); - rangeBitmap.add(0, upperEntryId + 1); + addRange(rangeBitmap, 0, upperEntryId + 1); } } else { LongBitmap rangeBitmap = rangeBitmapMap.computeIfAbsent(lowerLedgerId, k -> LongBitmaps.create()); - rangeBitmap.add(lowerEntryId, upperEntryId + 1); + addRange(rangeBitmap, lowerEntryId, upperEntryId + 1); } invalidateCaches(); } + private void addRange(LongBitmap bitmap, long from, long to) { + long before = bitmap.cardinality(); + bitmap.add(from, to); + totalCardinality += bitmap.cardinality() - before; + } + + private void removeRange(LongBitmap bitmap, long from, long to) { + long before = bitmap.cardinality(); + bitmap.remove(from, to); + totalCardinality -= before - bitmap.cardinality(); + } + + private void clearSubMapCardinality(Long2ObjectSortedMap subMap) { + for (LongBitmap bitmap : subMap.values()) { + totalCardinality -= bitmap.cardinality(); + } + subMap.clear(); + } + @Override public boolean contains(long ledgerId, long entryId) { LongBitmap rangeBitmap = rangeBitmapMap.get(ledgerId); @@ -175,6 +198,7 @@ public boolean isEmpty() { @Override public void clear() { rangeBitmapMap.clear(); + totalCardinality = 0; resetDirtyKeys(); invalidateCaches(); } @@ -271,14 +295,29 @@ public Range lastRange() { @Override public Map toRanges(int maxRanges) { + return toRanges(maxRanges, null); + } + + /** + * Serializes the individual-deleted positions as per-ledger bitmaps, including ledgers only while + * the cumulative cardinality stays within {@code maxRanges}. Ledgers beyond the limit are skipped + * silently (their ack state will be lost on broker restart); when {@code truncationNotifier} is + * provided it is invoked once with the number of skipped ledgers so callers can record the loss. + */ + public Map toRanges(int maxRanges, java.util.function.IntConsumer truncationNotifier) { Map internalBitSetMap = new HashMap<>(); MutableInt rangeCount = new MutableInt(); + MutableInt skippedLedgers = new MutableInt(); rangeBitmapMap.forEach((ledgerId, bitmap) -> { if (rangeCount.addAndGet((int) bitmap.cardinality()) > maxRanges) { + skippedLedgers.increment(); return; } internalBitSetMap.put(ledgerId, bitmap.serializeToLongArray()); }); + if (truncationNotifier != null && skippedLedgers.intValue() > 0) { + truncationNotifier.accept(skippedLedgers.intValue()); + } return internalBitSetMap; } @@ -290,6 +329,20 @@ public void build(Map internalRange) { internalRange.forEach((ledgerId, ranges) -> { rangeBitmapMap.put(ledgerId.longValue(), LongBitmaps.deserializeFromLongArray(ranges)); }); + recomputeTotalCardinality(); + invalidateCaches(); + } + + void buildFromBitmaps(Map bitmaps) { + rangeBitmapMap.clear(); + resetDirtyKeys(); + bitmaps.forEach((ledgerId, bytes) -> { + if (bytes != null && bytes.length > 0) { + rangeBitmapMap.put(ledgerId.longValue(), + LongBitmaps.deserialize(Unpooled.wrappedBuffer(bytes))); + } + }); + recomputeTotalCardinality(); invalidateCaches(); } @@ -379,8 +432,11 @@ void add(Range range) { ? getSafeEntry(upperEndpoint) : getSafeEntry(upperEndpoint) + 1; - rangeBitmapMap.computeIfAbsent(lowerEndpoint.getLedgerId(), k -> LongBitmaps.create()) - .add(lowerEntryIdOpen + 1); + LongBitmap bitmap = rangeBitmapMap.computeIfAbsent(lowerEndpoint.getLedgerId(), + k -> LongBitmaps.create()); + if (bitmap.checkedAdd(lowerEntryIdOpen + 1)) { + totalCardinality++; + } addOpenClosed(lowerEndpoint.getLedgerId(), lowerEntryIdOpen, upperEndpoint.getLedgerId(), upperEntryIdClosed); } @@ -408,13 +464,13 @@ void remove(Range range) { boolean sameLedger = lowerLedgerId == upperLedgerId; if (lowerIsEarliest) { - rangeBitmapMap.headMap(upperLedgerId).clear(); + clearSubMapCardinality(rangeBitmapMap.headMap(upperLedgerId)); } if (upperIsLatest) { - rangeBitmapMap.tailMap(lowerLedgerId + 1).clear(); + clearSubMapCardinality(rangeBitmapMap.tailMap(lowerLedgerId + 1)); } if (!sameLedger && !lowerIsEarliest && !upperIsLatest) { - rangeBitmapMap.subMap(lowerLedgerId + 1, upperLedgerId).clear(); + clearSubMapCardinality(rangeBitmapMap.subMap(lowerLedgerId + 1, upperLedgerId)); } LongBitmap lowerSet = lowerIsEarliest ? null : rangeBitmapMap.get(lowerLedgerId); @@ -422,13 +478,13 @@ void remove(Range range) { : (sameLedger ? lowerSet : rangeBitmapMap.get(upperLedgerId)); if (sameLedger && lowerSet != null) { - lowerSet.remove(lowerEntryId, upperEntryId + 1); + removeRange(lowerSet, lowerEntryId, upperEntryId + 1); } else { if (lowerSet != null) { - lowerSet.remove(lowerEntryId, lastPresentValue(lowerSet)); + removeRange(lowerSet, lowerEntryId, lastPresentValue(lowerSet)); } if (upperSet != null) { - upperSet.remove(0, upperEntryId + 1); + removeRange(upperSet, 0, upperEntryId + 1); } } @@ -450,28 +506,65 @@ boolean isDirtyLedgers(long ledgerId) { return ledgerId >= 0 && ledgerId <= Integer.MAX_VALUE && dirtyLedgers.contains(ledgerId); } + /** + * Atomically snapshots the dirty ledger IDs and clears the dirty set. + * + *

The returned set is consumed by the persistence layer to decide which ledgers + * need fresh AckState entries written this flush. Keys above {@code Integer.MAX_VALUE} + * are skipped at mark-time, so callers see raw ledger IDs in {@code [0, 2^31-1]}. + */ + Set snapshotAndClearDirtyLedgers() { + Set snapshot = new HashSet<>(); + dirtyLedgers.forEachLong(bit -> snapshot.add(bit - 1)); + dirtyLedgers.clear(); + return snapshot; + } + + void restoreDirtyLedgers(Set ledgers) { + for (long id : ledgers) { + if (id >= 0 && id < Integer.MAX_VALUE) { + dirtyLedgers.add(id + 1); + } + } + } + + /** + * Returns the serialized RoaringBitmap bytes for a specific ledger's individual acks, + * or {@code null} when the ledger has no entries in this set. + */ + byte[] bitmapOf(long ledgerId) { + LongBitmap bitmap = rangeBitmapMap.get(ledgerId); + return bitmap == null ? null : bitmap.serialize(); + } + + /** + * Invokes the given consumer for each ledger ID that currently has at least one + * individual-deleted entry in this set. Used by the persistence layer to enumerate + * the active ledgers that need an AckState or AckStateRef in the checkpoint. + */ + void forEachActiveLedger(LongConsumer action) { + rangeBitmapMap.keySet().forEach(action); + } + + /** + * Marks a single ledger as dirty. Used for batch-index deletions, which are tracked in + * {@code batchDeletedIndexes} rather than in the individual-deleted range bitmap. + */ + void markDirtyLedger(long ledgerId) { + markDirty(ledgerId, ledgerId); + } + private void markDirty(long lowerLedgerId, long upperLedgerId) { - // Original semantics: dirtyLedgers.addOpenClosed(k1, 0, k2, 0), which in LongPair ordering - // is (k1, k2] on ledger ids. LongBitmap.add(from, to) is half-open [from, to), so shift both - // bounds. Same-ledger or inverted range is a no-op. - // - // Note: Ledger IDs are 64-bit longs, but LongBitmap supports unsigned 32-bit range [0, 2^32-1]. - // In practice, BookKeeper ledger IDs rarely exceed Integer.MAX_VALUE. If upperLedgerId exceeds - // this limit, we skip tracking to avoid overflow. This is acceptable because: - // 1. The dirty tracker is an optimization hint for selective persistence - // 2. Missing a dirty mark means conservative full-ledger write (safe, just slower) - // 3. Real-world ledger IDs stay well within 32-bit range - if (upperLedgerId <= lowerLedgerId || lowerLedgerId < 0) { + if (upperLedgerId < lowerLedgerId || lowerLedgerId < 0) { return; } if (lowerLedgerId >= Integer.MAX_VALUE || upperLedgerId > Integer.MAX_VALUE) { - log.warn() - .attr("lowerLedgerId", lowerLedgerId) - .attr("upperLedgerId", upperLedgerId) - .log("Skipping dirty tracking for ledger ID at/exceeding Integer.MAX_VALUE"); + log.warn("Skipping dirty tracking for ledger ID at/exceeding Integer.MAX_VALUE, " + + "lowerLedgerId: {}, upperLedgerId: {}", + lowerLedgerId, upperLedgerId); return; } - dirtyLedgers.add(lowerLedgerId + 1, upperLedgerId + 1); + dirtyLedgers.add(lowerLedgerId + 1, upperLedgerId + 2); } private boolean isValid(long ledgerId, long entryId) { @@ -491,4 +584,16 @@ private void invalidateCaches() { updatedAfterCachedForSize = true; updatedAfterCachedForToString = true; } + + long totalCardinality() { + return totalCardinality; + } + + private void recomputeTotalCardinality() { + long total = 0; + for (LongBitmap bitmap : rangeBitmapMap.values()) { + total += bitmap.cardinality(); + } + totalCardinality = total; + } } diff --git a/managed-ledger/src/main/proto/MLDataFormats.proto b/managed-ledger/src/main/proto/MLDataFormats.proto index 825da1e414cea..4109fcf36130a 100644 --- a/managed-ledger/src/main/proto/MLDataFormats.proto +++ b/managed-ledger/src/main/proto/MLDataFormats.proto @@ -140,6 +140,45 @@ message ManagedCursorInfo { // Additional custom properties associated with // the cursor repeated StringProperty cursorProperties = 8; + + optional int64 lastCmCursorLedgerId = 9; + optional int64 lastCmEntryId = 10; +} + +message CursorLogEntry { + oneof entry { + CursorCheckpoint checkpoint = 2; + CursorCheckpointChunk checkpointChunk = 3; + } +} + +// A consistent snapshot of cursor ack state for one persisted ledger update. +// Each checkpoint entry contains exactly one inline ackState; other active +// ledgers are represented by refs to their latest persisted checkpoints. +message CursorCheckpoint { + required int64 markDeleteLedgerId = 1; + required int64 markDeleteEntryId = 2; + repeated LongProperty properties = 3; + repeated AckState ackStates = 4; + repeated AckStateRef ackStateRefs = 5; +} + +message AckState { + required int64 msgLedgerId = 1; + optional bytes ackBitmap = 2; + repeated BatchedEntryDeletionIndexInfo batchAcks = 3; +} + +message AckStateRef { + required int64 msgLedgerId = 1; + required int64 cursorLedgerId = 2; + required int64 entryId = 3; +} + +message CursorCheckpointChunk { + optional int32 partIndex = 1; + optional int32 partCount = 2; + optional bytes checkpointBytes = 3; } enum CompressionType { diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointLogRecoveryTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointLogRecoveryTest.java new file mode 100644 index 0000000000000..e73f41fc930a3 --- /dev/null +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointLogRecoveryTest.java @@ -0,0 +1,355 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.impl; + +import static org.assertj.core.api.Assertions.assertThat; +import com.google.protobuf.ByteString; +import java.util.concurrent.TimeUnit; +import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.proto.MLDataFormats.AckState; +import org.apache.bookkeeper.mledger.proto.MLDataFormats.CursorCheckpoint; +import org.apache.bookkeeper.mledger.proto.MLDataFormats.CursorCheckpointChunk; +import org.apache.bookkeeper.mledger.proto.MLDataFormats.CursorLogEntry; +import org.apache.bookkeeper.mledger.proto.MLDataFormats.PositionInfo; +import org.apache.bookkeeper.test.MockedBookKeeperTestCase; +import org.testng.annotations.Test; + +public class CursorCheckpointLogRecoveryTest extends MockedBookKeeperTestCase { + + private LedgerHandle createLedger() throws Exception { + return bkc.createLedger(BookKeeper.DigestType.MAC, new byte[0]); + } + + private long appendRaw(LedgerHandle lh, byte[] data) throws Exception { + return lh.append(data); + } + + private byte[] wrapCheckpoint(CursorCheckpoint cp) throws Exception { + return CursorLogEntry.newBuilder() + .setCheckpoint(CursorCheckpoint.parseFrom(cp.toByteArray())) + .build() + .toByteArray(); + } + + private byte[] wrapChunkPart(int partIndex, int partCount, byte[] payload) { + return CursorLogEntry.newBuilder() + .setCheckpointChunk(CursorCheckpointChunk.newBuilder() + .setPartIndex(partIndex).setPartCount(partCount) + .setCheckpointBytes(ByteString.copyFrom(payload)) + .build()) + .build() + .toByteArray(); + } + + private byte[] wrapLegacyPositionInfo(long ledgerId, long entryId) { + return PositionInfo.newBuilder() + .setLedgerId(ledgerId).setEntryId(entryId) + .build() + .toByteArray(); + } + + private CursorCheckpoint makeCheckpoint(long mdLedgerId, long mdEntryId) { + return CursorCheckpoint.newBuilder() + .setMarkDeleteLedgerId(mdLedgerId) + .setMarkDeleteEntryId(mdEntryId) + .addAckStates(AckState.newBuilder().setMsgLedgerId(mdLedgerId).build()) + .build(); + } + + /** + * Scenario: checkpoint → legacy → chunk(complete) → chunk(incomplete) → chunk(incomplete) + * + * Recovery reads last (incomplete) → scanBack → finds chunk(complete) → assembles. + */ + @Test(timeOut = 30000) + public void testScanBackFindsCompleteChunk() throws Exception { + LedgerHandle lh = createLedger(); + CursorCheckpointLog writer = new CursorCheckpointLog(5 * 1024 * 1024); + + appendRaw(lh, wrapCheckpoint(makeCheckpoint(10, 0))); + appendRaw(lh, wrapLegacyPositionInfo(10, 1)); + // Write a complete 2-part chunk + byte[] cpBytes = makeCheckpoint(30, 5).toByteArray(); + int half = cpBytes.length / 2; + byte[] part0 = new byte[half]; + byte[] part1 = new byte[cpBytes.length - half]; + System.arraycopy(cpBytes, 0, part0, 0, half); + System.arraycopy(cpBytes, half, part1, 0, part1.length); + appendRaw(lh, wrapChunkPart(0, 2, part0)); + appendRaw(lh, wrapChunkPart(1, 2, part1)); + // Two incomplete chunks after + appendRaw(lh, wrapChunkPart(0, 3, new byte[100])); + appendRaw(lh, wrapChunkPart(1, 3, new byte[100])); + + CursorCheckpointLog.RecoveredState state = writer.readLatest(lh).get(5, TimeUnit.SECONDS); + assertThat(state.isLegacy()).isFalse(); + assertThat(state.checkpoint.getMarkDeleteLedgerId()).isEqualTo(30L); + } + + /** + * Scenario: checkpoint → legacy → chunk(incomplete) + * + * Recovery reads last (incomplete) → scanBack → finds legacy → returns legacy. + */ + @Test(timeOut = 30000) + public void testScanBackFallsBackToLegacy() throws Exception { + LedgerHandle lh = createLedger(); + CursorCheckpointLog writer = new CursorCheckpointLog(5 * 1024 * 1024); + + appendRaw(lh, wrapCheckpoint(makeCheckpoint(10, 0))); + appendRaw(lh, wrapLegacyPositionInfo(20, 5)); + // Incomplete chunk + appendRaw(lh, wrapChunkPart(0, 3, new byte[100])); + + CursorCheckpointLog.RecoveredState state = writer.readLatest(lh).get(5, TimeUnit.SECONDS); + assertThat(state.isLegacy()).isTrue(); + PositionInfo pi = PositionInfo.parseFrom(state.legacyBytes); + assertThat(pi.getLedgerId()).isEqualTo(20L); + assertThat(pi.getEntryId()).isEqualTo(5L); + } + + /** + * Scenario: only incomplete chunks, scanBack exhausts, error. + */ + @Test(timeOut = 30000) + public void testScanBackExhausts() throws Exception { + LedgerHandle lh = createLedger(); + CursorCheckpointLog writer = new CursorCheckpointLog(5 * 1024 * 1024); + + appendRaw(lh, wrapChunkPart(0, 3, new byte[100])); + appendRaw(lh, wrapChunkPart(1, 3, new byte[100])); + + try { + writer.readLatest(lh).get(5, TimeUnit.SECONDS); + assertThat(false).as("Should have failed").isTrue(); + } catch (Exception e) { + assertThat(e).hasCauseInstanceOf(ManagedLedgerException.class); + } + } + + /** + * Scenario: last entry is legacy → returns directly (no scanBack). + */ + @Test(timeOut = 30000) + public void testLegacyAsLastEntry() throws Exception { + LedgerHandle lh = createLedger(); + CursorCheckpointLog writer = new CursorCheckpointLog(5 * 1024 * 1024); + + appendRaw(lh, wrapCheckpoint(makeCheckpoint(10, 0))); + appendRaw(lh, wrapLegacyPositionInfo(20, 5)); + + CursorCheckpointLog.RecoveredState state = writer.readLatest(lh).get(5, TimeUnit.SECONDS); + assertThat(state.isLegacy()).isTrue(); + } + + /** + * Scenario: last entry is a valid checkpoint → returns directly. + */ + @Test(timeOut = 30000) + public void testCheckpointAsLastEntry() throws Exception { + LedgerHandle lh = createLedger(); + CursorCheckpointLog writer = new CursorCheckpointLog(5 * 1024 * 1024); + + appendRaw(lh, wrapLegacyPositionInfo(10, 0)); + appendRaw(lh, wrapCheckpoint(makeCheckpoint(20, 5))); + + CursorCheckpointLog.RecoveredState state = writer.readLatest(lh).get(5, TimeUnit.SECONDS); + assertThat(state.isLegacy()).isFalse(); + assertThat(state.checkpoint.getMarkDeleteLedgerId()).isEqualTo(20L); + } + + /** + * Scenario: old, chunk(complete as last entry). + */ + @Test(timeOut = 30000) + public void testCompleteChunkAsLastEntry() throws Exception { + LedgerHandle lh = createLedger(); + CursorCheckpointLog writer = new CursorCheckpointLog(5 * 1024 * 1024); + + appendRaw(lh, wrapLegacyPositionInfo(10, 0)); + byte[] cpBytes = makeCheckpoint(40, 7).toByteArray(); + int half = cpBytes.length / 2; + byte[] part0 = new byte[half]; + byte[] part1 = new byte[cpBytes.length - half]; + System.arraycopy(cpBytes, 0, part0, 0, half); + System.arraycopy(cpBytes, half, part1, 0, part1.length); + appendRaw(lh, wrapChunkPart(0, 2, part0)); + appendRaw(lh, wrapChunkPart(1, 2, part1)); + + CursorCheckpointLog.RecoveredState state = writer.readLatest(lh).get(5, TimeUnit.SECONDS); + assertThat(state.isLegacy()).isFalse(); + assertThat(state.checkpoint.getMarkDeleteLedgerId()).isEqualTo(40L); + } + + /** + * Scenario: chunk(complete), chunk(incomplete). + */ + @Test(timeOut = 30000) + public void testCompleteThenIncompleteChunkFallsBackToPreviousCompleteChunk() throws Exception { + LedgerHandle lh = createLedger(); + CursorCheckpointLog writer = new CursorCheckpointLog(5 * 1024 * 1024); + + byte[] cpBytes = makeCheckpoint(50, 9).toByteArray(); + int half = cpBytes.length / 2; + byte[] part0 = new byte[half]; + byte[] part1 = new byte[cpBytes.length - half]; + System.arraycopy(cpBytes, 0, part0, 0, half); + System.arraycopy(cpBytes, half, part1, 0, part1.length); + appendRaw(lh, wrapChunkPart(0, 2, part0)); + appendRaw(lh, wrapChunkPart(1, 2, part1)); + appendRaw(lh, wrapChunkPart(0, 3, new byte[64])); + + CursorCheckpointLog.RecoveredState state = writer.readLatest(lh).get(5, TimeUnit.SECONDS); + assertThat(state.isLegacy()).isFalse(); + assertThat(state.checkpoint.getMarkDeleteLedgerId()).isEqualTo(50L); + } + + /** + * Scenario: new, chunk(incomplete). + */ + @Test(timeOut = 30000) + public void testNewThenIncompleteChunkFallsBackToNewCheckpoint() throws Exception { + LedgerHandle lh = createLedger(); + CursorCheckpointLog writer = new CursorCheckpointLog(5 * 1024 * 1024); + + appendRaw(lh, wrapCheckpoint(makeCheckpoint(60, 11))); + appendRaw(lh, wrapChunkPart(0, 3, new byte[64])); + + CursorCheckpointLog.RecoveredState state = writer.readLatest(lh).get(5, TimeUnit.SECONDS); + assertThat(state.isLegacy()).isFalse(); + assertThat(state.checkpoint.getMarkDeleteLedgerId()).isEqualTo(60L); + } + + /** + * Boundary: recovery keeps scanning back, but only within MAX_SCAN_BACK (1000). + */ + @Test(timeOut = 30000) + public void testScanBackLimitBoundary() throws Exception { + LedgerHandle lh = createLedger(); + CursorCheckpointLog writer = new CursorCheckpointLog(5 * 1024 * 1024); + + appendRaw(lh, wrapCheckpoint(makeCheckpoint(70, 13))); + for (int i = 0; i < 1000; i++) { + appendRaw(lh, wrapChunkPart(0, 3, new byte[16])); + } + CursorCheckpointLog.RecoveredState withinLimit = writer.readLatest(lh).get(5, TimeUnit.SECONDS); + assertThat(withinLimit.isLegacy()).isFalse(); + assertThat(withinLimit.checkpoint.getMarkDeleteLedgerId()).isEqualTo(70L); + + appendRaw(lh, wrapChunkPart(0, 3, new byte[16])); + try { + writer.readLatest(lh).get(5, TimeUnit.SECONDS); + assertThat(false).as("Should fail after exceeding scan-back limit").isTrue(); + } catch (Exception e) { + assertThat(e).hasCauseInstanceOf(ManagedLedgerException.class); + } + } + + /** + * Scenario: entry parses as CursorLogEntry but contains no known payload. + * + * Recovery should fail fast because this is a data error. + */ + @Test(timeOut = 30000) + public void testInvalidEnvelopeFailsRecovery() throws Exception { + LedgerHandle lh = createLedger(); + CursorCheckpointLog writer = new CursorCheckpointLog(5 * 1024 * 1024); + + appendRaw(lh, CursorLogEntry.getDefaultInstance().toByteArray()); + + try { + writer.readLatest(lh).get(5, TimeUnit.SECONDS); + assertThat(false).as("Should have failed").isTrue(); + } catch (Exception e) { + assertThat(e).hasCauseInstanceOf(ManagedLedgerException.class); + } + } + + /** + * Scenario: bytes cannot be parsed as CursorLogEntry or PositionInfo. + * + * Recovery should fail rather than treating corrupted data as legacy. + */ + @Test(timeOut = 30000) + public void testCorruptedBytesFailRecovery() throws Exception { + LedgerHandle lh = createLedger(); + CursorCheckpointLog writer = new CursorCheckpointLog(5 * 1024 * 1024); + + appendRaw(lh, new byte[]{(byte) 0x80}); + + try { + writer.readLatest(lh).get(5, TimeUnit.SECONDS); + assertThat(false).as("Should have failed").isTrue(); + } catch (Exception e) { + assertThat(e).hasCauseInstanceOf(ManagedLedgerException.class); + } + } + + /** + * Scenario: latest chunk looks complete, but assembly fails. + * + * Recovery should continue scan-back and return the previous valid checkpoint. + */ + @Test(timeOut = 30000) + public void testBrokenCompleteChunkFallsBackToPreviousCheckpoint() throws Exception { + LedgerHandle lh = createLedger(); + CursorCheckpointLog writer = new CursorCheckpointLog(5 * 1024 * 1024); + + appendRaw(lh, wrapCheckpoint(makeCheckpoint(80, 15))); + appendRaw(lh, new byte[]{0x01, 0x02, 0x03}); // not a valid chunk part + appendRaw(lh, wrapChunkPart(1, 2, new byte[]{0x11, 0x22})); + + CursorCheckpointLog.RecoveredState state = writer.readLatest(lh).get(5, TimeUnit.SECONDS); + assertThat(state.isLegacy()).isFalse(); + assertThat(state.checkpoint.getMarkDeleteLedgerId()).isEqualTo(80L); + } + + /** + * Scenario: readAt on the last part of a chunked checkpoint returns the assembled checkpoint, + * while readAt on a non-terminal part fails fast (ref targets must not scan back). + */ + @Test(timeOut = 30000) + public void testReadAtAssemblesChunkAndFailsOnNonTerminalPart() throws Exception { + LedgerHandle lh = createLedger(); + CursorCheckpointLog writer = new CursorCheckpointLog(5 * 1024 * 1024); + + byte[] cpBytes = makeCheckpoint(90, 21).toByteArray(); + int half = cpBytes.length / 2; + byte[] part0 = new byte[half]; + byte[] part1 = new byte[cpBytes.length - half]; + System.arraycopy(cpBytes, 0, part0, 0, half); + System.arraycopy(cpBytes, half, part1, 0, part1.length); + long firstEntry = appendRaw(lh, wrapChunkPart(0, 2, part0)); + long lastEntry = appendRaw(lh, wrapChunkPart(1, 2, part1)); + + CursorCheckpointLog.RecoveredState state = writer.readAt(lh, lastEntry).get(5, TimeUnit.SECONDS); + assertThat(state.isLegacy()).isFalse(); + assertThat(state.checkpoint.getMarkDeleteLedgerId()).isEqualTo(90L); + assertThat(state.commitEntryId).isEqualTo(lastEntry); + + try { + writer.readAt(lh, firstEntry).get(5, TimeUnit.SECONDS); + assertThat(false).as("Should have failed on a non-terminal chunk part").isTrue(); + } catch (Exception e) { + assertThat(e).hasCauseInstanceOf(ManagedLedgerException.class); + } + } +} diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointPersistenceOrderingTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointPersistenceOrderingTest.java new file mode 100644 index 0000000000000..241a5dbab64c2 --- /dev/null +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointPersistenceOrderingTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.impl; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.testng.annotations.Test; + +/** + * Ordering guarantees of {@link CursorCheckpointPersistence}: the per-msg-ledger checkpoint + * position index must never regress, otherwise a later AckStateRef would point at an older + * (smaller) ack bitmap and acks recorded in the newer checkpoint would be lost on recovery. + */ +public class CursorCheckpointPersistenceOrderingTest { + + private CursorCheckpointPersistence newInstance() { + return new CursorCheckpointPersistence(null, new ReentrantReadWriteLock(), + null, null, new byte[0], true); + } + + @Test + public void testRecordCheckpointPosMonotonic() { + CursorCheckpointPersistence persistence = newInstance(); + assertNull(persistence.checkpointPosOf(5)); + + // First record wins by absence. + persistence.recordCheckpointPos(5, pos(1, 10)); + assertEquals(persistence.checkpointPosOf(5), pos(1, 10)); + + // Newer position advances the index. + persistence.recordCheckpointPos(5, pos(1, 20)); + assertEquals(persistence.checkpointPosOf(5), pos(1, 20)); + + // A stale completion (out-of-order append callback, same cursor ledger) must not regress. + persistence.recordCheckpointPos(5, pos(1, 15)); + assertEquals(persistence.checkpointPosOf(5), pos(1, 20)); + + // A position from an older cursor ledger (after rollover replay) must not regress either. + persistence.recordCheckpointPos(5, pos(0, 999)); + assertEquals(persistence.checkpointPosOf(5), pos(1, 20)); + + // Ledger ids are independent entries. + persistence.recordCheckpointPos(7, pos(1, 3)); + assertEquals(persistence.checkpointPosOf(7), pos(1, 3)); + assertEquals(persistence.checkpointPosOf(5), pos(1, 20)); + } + + private static Position pos(long ledgerId, long entryId) { + return PositionFactory.create(ledgerId, entryId); + } +} diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java index 631437f89d9e9..cdee6454d5df0 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java @@ -127,6 +127,7 @@ import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.pulsar.common.api.proto.CommandSubscribe; import org.apache.pulsar.common.api.proto.IntRange; +import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.BitSetRecyclable; import org.apache.pulsar.common.util.collections.LongPairRangeSet; @@ -6243,4 +6244,1265 @@ public void asyncOpenLedger(final long lId, final DigestType digestType, final b } private static final Logger log = LoggerFactory.getLogger(ManagedCursorTest.class); + @Test(timeOut = 30000) + public void testCheckpointIndividualAckPersistAndRecover() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true); + config.setMaxUnackedRangesToPersistInMetadataStore(-1); + config.setThrottleMarkDelete(0); + + String ledgerName = "test_checkpoint_individual_ack"; + ManagedLedger ledger = factory.open(ledgerName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + List positions = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + positions.add(ledger.addEntry(("msg-" + i).getBytes(Encoding))); + } + + cursor.markDelete(positions.get(0)); + cursor.delete(positions.get(2)); + cursor.delete(positions.get(5)); + + assertThat(cursor.isMessageDeleted(positions.get(2))).isTrue(); + assertThat(cursor.isMessageDeleted(positions.get(5))).isTrue(); + assertThat(cursor.isMessageDeleted(positions.get(3))).isFalse(); + + ManagedCursorImpl cursorBeforeClose = cursor; + Awaitility.await().untilAsserted( + () -> assertThat(cursorBeforeClose.getCursorLedger()).isNotNegative()); + ledger.close(); + + ledger = factory.open(ledgerName, config); + cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + assertThat(cursor.getMarkDeletedPosition()).isEqualTo(positions.get(0)); + assertThat(cursor.isMessageDeleted(positions.get(2))).isTrue(); + assertThat(cursor.isMessageDeleted(positions.get(5))).isTrue(); + assertThat(cursor.isMessageDeleted(positions.get(3))).isFalse(); + ledger.close(); + } + + @Test(timeOut = 30000) + public void testCheckpointMultiMsgLedgerRecovery() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true); + config.setMaxEntriesPerLedger(5); // small msg-ledger → multiple msgLedgers + config.setMaxUnackedRangesToPersistInMetadataStore(-1); + config.setThrottleMarkDelete(0); + + String ledgerName = "test_checkpoint_multi_ledger"; + ManagedLedger ledger = factory.open(ledgerName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + List positions = new ArrayList<>(); + for (int i = 0; i < 20; i++) { + positions.add(ledger.addEntry(("msg-" + i).getBytes(Encoding))); + } + + cursor.markDelete(positions.get(0)); + cursor.delete(positions.get(3)); + cursor.delete(positions.get(7)); + cursor.delete(positions.get(12)); + cursor.delete(positions.get(18)); + + ManagedCursorImpl cursorBeforeClose2 = cursor; + Awaitility.await().untilAsserted( + () -> assertThat(cursorBeforeClose2.getCursorLedger()).isNotNegative()); + ledger.close(); + + ledger = factory.open(ledgerName, config); + cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + assertThat(cursor.getMarkDeletedPosition()).isEqualTo(positions.get(0)); + assertThat(cursor.isMessageDeleted(positions.get(3))).isTrue(); + assertThat(cursor.isMessageDeleted(positions.get(7))).isTrue(); + assertThat(cursor.isMessageDeleted(positions.get(12))).isTrue(); + assertThat(cursor.isMessageDeleted(positions.get(18))).isTrue(); + assertThat(cursor.isMessageDeleted(positions.get(5))).isFalse(); + ledger.close(); + } + + @Test(timeOut = 30000) + public void testCheckpointCursorLedgerRollover() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true); + config.setMaxEntriesPerLedger(5); + config.setMetadataMaxEntriesPerLedger(5); // force frequent cursor-ledger rollover + config.setMaxUnackedRangesToPersistInMetadataStore(-1); + config.setThrottleMarkDelete(0); + + String ledgerName = "test_checkpoint_rollover"; + ManagedLedger ledger = factory.open(ledgerName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + + List positions = new ArrayList<>(); + for (int i = 0; i < 30; i++) { + positions.add(ledger.addEntry(("msg-" + i).getBytes(Encoding))); + cursor.markDelete(positions.get(i)); + } + cursor.delete(positions.get(5)); + cursor.delete(positions.get(15)); + cursor.delete(positions.get(25)); + + ManagedCursorImpl cursorBeforeClose3 = cursor; + Awaitility.await().untilAsserted( + () -> assertThat(cursorBeforeClose3.getCursorLedger()).isNotNegative()); + ledger.close(); + + ledger = factory.open(ledgerName, config); + cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + assertThat(cursor.isMessageDeleted(positions.get(5))).isTrue(); + assertThat(cursor.isMessageDeleted(positions.get(15))).isTrue(); + assertThat(cursor.isMessageDeleted(positions.get(25))).isTrue(); + ledger.close(); + } + + @Test(timeOut = 60000) + public void testCheckpointImmediateCloseFlushesAllAppliedIndividualAcks() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true); + config.setMaxUnackedRangesToPersistInMetadataStore(-1); + config.setThrottleMarkDelete(0); + + String ledgerName = "test_checkpoint_immediate_close"; + ManagedLedger ledger = factory.open(ledgerName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + + List positions = new ArrayList<>(); + for (int i = 0; i < 1000; i++) { + positions.add(ledger.addEntry(("msg-" + i).getBytes(Encoding))); + } + cursor.markDelete(positions.get(0)); + // Individual-delete even positions without waiting for any periodic flush. + for (int i = 2; i < 1000; i += 2) { + cursor.delete(positions.get(i)); + } + + // Close immediately: no Awaitility wait for the periodic flush. All acks already + // applied to individualDeletedMessages must be captured by the closing persist. + ledger.close(); + + ledger = factory.open(ledgerName, config); + cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + assertThat(cursor.getMarkDeletedPosition()).isEqualTo(positions.get(0)); + int lost = 0; + for (int i = 2; i < 1000; i += 2) { + if (!cursor.isMessageDeleted(positions.get(i))) { + lost++; + } + } + assertThat(lost).isZero(); + for (int i = 1; i < 1000; i += 2) { + assertThat(cursor.isMessageDeleted(positions.get(i))).isFalse(); + } + ledger.close(); + } + + @Test(timeOut = 60000) + public void testCheckpointChunkedPersistAndRecoverLargeAckSet() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true); + // Force the checkpoint log to chunk entries: a single per-ledger bitmap of + // 5k alternating positions (~10KB Roaring) far exceeds this cap. + config.setPersistentUnackedRangesMaxEntrySize(2048); + config.setMaxUnackedRangesToPersistInMetadataStore(-1); + config.setThrottleMarkDelete(0); + + String ledgerName = "test_checkpoint_chunked_large_ack_set"; + ManagedLedger ledger = factory.open(ledgerName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + + final int total = 5_000; + List positions = new ArrayList<>(total); + for (int i = 0; i < total; i++) { + positions.add(ledger.addEntry(("msg-" + i).getBytes(Encoding))); + } + cursor.markDelete(positions.get(0)); + for (int i = 2; i < total; i += 2) { + cursor.delete(positions.get(i)); + } + + ledger.close(); + + ledger = factory.open(ledgerName, config); + cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + assertThat(cursor.getMarkDeletedPosition()).isEqualTo(positions.get(0)); + int lost = 0; + for (int i = 2; i < total; i += 2) { + if (!cursor.isMessageDeleted(positions.get(i))) { + lost++; + } + } + assertThat(lost).isZero(); + assertThat(cursor.isMessageDeleted(positions.get(1))).isFalse(); + ledger.close(); + } + + @Test(timeOut = 60000) + public void testCheckpointGcAfterMarkDeleteAbsorbsAllAcks() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true); + config.setMaxEntriesPerLedger(2); + config.setMetadataMaxEntriesPerLedger(2); // frequent rollovers + config.setMaxUnackedRangesToPersistInMetadataStore(-1); + config.setThrottleMarkDelete(0); + + String ledgerName = "test_checkpoint_gc"; + ManagedLedger ledger = factory.open(ledgerName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + + List positions = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + positions.add(ledger.addEntry(("m-" + i).getBytes(Encoding))); + } + cursor.markDelete(positions.get(0)); + cursor.delete(positions.get(1)); + cursor.markDelete(positions.get(2)); + cursor.delete(positions.get(3)); + + Set seenCursorLedgers = new HashSet<>(); + seenCursorLedgers.add(cursor.getCursorLedger()); + for (int i = 0; i < 20; i++) { + Position p = ledger.addEntry(("extra-" + i).getBytes(Encoding)); + cursor.markDelete(p); + seenCursorLedgers.add(cursor.getCursorLedger()); + } + assertThat(seenCursorLedgers.size()).isGreaterThan(1); + + // At least one old cursor ledger should be GC'd (maxHold absorbed). Invoke the GC + // explicitly instead of relying on the incidental rollover timing: this keeps the + // test deterministic regardless of executor scheduling in shared-JVM test runs. + cursor.gcOldCursorLedgers(); + Set seenSnapshot = Collections.unmodifiableSet(new HashSet<>(seenCursorLedgers)); + ManagedCursorImpl cursorRef = cursor; + Awaitility.await().untilAsserted(() -> { + Set live = bkc.getLedgers(); + long currentId = cursorRef.getCursorLedger(); + boolean anyReclaimed = seenSnapshot.stream() + .anyMatch(id -> id != currentId && !live.contains(id)); + assertThat(anyReclaimed).isTrue(); + }); + ledger.close(); + } + + @Test(timeOut = 60000) + public void testCheckpointGcRetriesAfterLedgerDeleteFailure() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true); + config.setMaxEntriesPerLedger(2); + config.setMetadataMaxEntriesPerLedger(2); // frequent rollovers -> frequent GC attempts + config.setMaxUnackedRangesToPersistInMetadataStore(-1); + config.setThrottleMarkDelete(0); + + String ledgerName = "test_checkpoint_gc_retry_on_delete_failure"; + ManagedLedger ledger = factory.open(ledgerName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + + List positions = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + positions.add(ledger.addEntry(("m-" + i).getBytes(Encoding))); + } + cursor.markDelete(positions.get(0)); + cursor.markDelete(positions.get(3)); // everything absorbed; all refs stale + + // Fail BK ledger deletes until the flag is cleared, via the mock's dedicated hook + // (ledger deletes do not go through the metadata store in this mock). + AtomicBoolean failLedgerDeletes = new AtomicBoolean(true); + ((org.apache.bookkeeper.client.PulsarMockBookKeeper) bkc).setDeleteLedgerFailure(failLedgerDeletes::get); + + Set seenBeforeFailure = new HashSet<>(); + for (int i = 0; i < 6; i++) { + Position p = ledger.addEntry(("failing-" + i).getBytes(Encoding)); + cursor.markDelete(p); + seenBeforeFailure.add(cursor.getCursorLedger()); + } + Set failingSnapshot = Collections.unmodifiableSet(new HashSet<>(seenBeforeFailure)); + ManagedCursorImpl cursorDuringFailure = cursor; + + // While deletes fail, old cursor ledgers must survive and GC must not lose its + // bookkeeping (they stay tracked for retry on the next rollover). + Awaitility.await().untilAsserted(() -> { + Set live = bkc.getLedgers(); + long currentId = cursorDuringFailure.getCursorLedger(); + boolean anyStillLive = failingSnapshot.stream() + .anyMatch(id -> id != currentId && live.contains(id)); + assertThat(anyStillLive).isTrue(); + }); + + // Lift the fault; the next rollovers retry the deletes and reclaim the ledgers. + failLedgerDeletes.set(false); + Set seenAfterRecovery = new HashSet<>(); + for (int i = 0; i < 8; i++) { + Position p = ledger.addEntry(("recovered-" + i).getBytes(Encoding)); + cursor.markDelete(p); + seenAfterRecovery.add(cursor.getCursorLedger()); + } + Set allSeen = new HashSet<>(failingSnapshot); + allSeen.addAll(seenAfterRecovery); + ManagedCursorImpl cursorAfterRecovery = cursor; + Awaitility.await().untilAsserted(() -> { + Set live = bkc.getLedgers(); + long currentId = cursorAfterRecovery.getCursorLedger(); + boolean anyReclaimed = allSeen.stream() + .anyMatch(id -> id != currentId && !live.contains(id)); + assertThat(anyReclaimed).isTrue(); + }); + ledger.close(); + } + + @Test(timeOut = 30000) + public void testCheckpointBatchAckPersistAndRecover() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true); + config.setMaxUnackedRangesToPersistInMetadataStore(-1); + config.setMaxBatchDeletedIndexToPersist(1000); + config.setDeletionAtBatchIndexLevelEnabled(true); + config.setThrottleMarkDelete(0); + + String ledgerName = "test_checkpoint_batch_ack"; + ManagedLedger ledger = factory.open(ledgerName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + List positions = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + positions.add(ledger.addEntry(("entry-" + i).getBytes(Encoding))); + } + + cursor.markDelete(positions.get(0)); + // Simulate batch ack: add to batchDeletedIndexes directly + Position batchPos = positions.get(3); + BitSet bitSet = new BitSet(32); + bitSet.set(2, 5); // ack batch indices 2,3,4 + cursor.batchDeletedIndexes.put(batchPos, bitSet); + + assertThat(cursor.getDeletedBatchIndexesAsLongArray(batchPos)).isNotEmpty(); + + cursor.markDelete(positions.get(1)); + + ManagedCursorImpl cursorBeforeClose4 = cursor; + Awaitility.await().untilAsserted( + () -> assertThat(cursorBeforeClose4.getCursorLedger()).isNotNegative()); + ledger.close(); + + ledger = factory.open(ledgerName, config); + cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + assertThat(cursor.getDeletedBatchIndexesAsLongArray(positions.get(3))).isNotEmpty(); + ledger.close(); + } + + @Test(timeOut = 30000) + public void testCheckpointLegacyFallback() throws Exception { + // When feature is off, legacy PositionInfo path is used. When the feature is then + // turned on and we reopen, the cursor should detect the legacy format and recover + // from PositionInfo. + String ledgerName = "test_checkpoint_legacy_fallback"; + + ManagedLedgerConfig legacyConfig = new ManagedLedgerConfig(); + legacyConfig.setMaxUnackedRangesToPersistInMetadataStore(-1); + legacyConfig.setThrottleMarkDelete(0); + ManagedLedger ledger = factory.open(ledgerName, legacyConfig); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + List positions = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + positions.add(ledger.addEntry(("msg-" + i).getBytes(Encoding))); + } + cursor.putCursorProperty("cp-key", "cp-value").get(); + cursor.markDelete(positions.get(0), Collections.singletonMap("md-key", 1L)); + cursor.delete(positions.get(2)); + ledger.close(); + + ManagedLedgerConfig newConfig = new ManagedLedgerConfig(); + newConfig.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true); + newConfig.setMaxUnackedRangesToPersistInMetadataStore(-1); + newConfig.setThrottleMarkDelete(0); + ledger = factory.open(ledgerName, newConfig); + cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + assertThat(cursor.getMarkDeletedPosition()).isEqualTo(positions.get(0)); + assertThat(cursor.isMessageDeleted(positions.get(2))).isTrue(); + assertThat(cursor.isMessageDeleted(positions.get(1))).isFalse(); + assertThat(cursor.getProperties()).containsEntry("md-key", 1L); + assertThat(cursor.getCursorProperties()).containsEntry("cp-key", "cp-value"); + ledger.close(); + } + + @Test(timeOut = 30000) + public void testCheckpointForwardCompatibilityNewToLegacyConfig() throws Exception { + // Start with per-msgLedger persistence enabled, persist checkpoint/ref state, then reopen with legacy config. + // Recovery should still read new-format cursor entries and rebuild cursor state. + String ledgerName = "test_checkpoint_new_to_legacy_config"; + + ManagedLedgerConfig newConfig = new ManagedLedgerConfig(); + newConfig.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true); + newConfig.setMaxEntriesPerLedger(3); + newConfig.setMetadataMaxEntriesPerLedger(3); + newConfig.setMaxUnackedRangesToPersistInMetadataStore(-1); + newConfig.setThrottleMarkDelete(0); + ManagedLedger ledger = factory.open(ledgerName, newConfig); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + + List positions = new ArrayList<>(); + for (int i = 0; i < 12; i++) { + positions.add(ledger.addEntry(("msg-" + i).getBytes(Encoding))); + } + cursor.putCursorProperty("cp-new", "cp-new-value").get(); + cursor.markDelete(positions.get(0)); + cursor.delete(positions.get(2)); + cursor.delete(positions.get(5)); + cursor.delete(positions.get(8)); + cursor.markDelete(positions.get(6), Collections.singletonMap("md-new", 6L)); + Thread.sleep(500); + ledger.close(); + + ManagedLedgerConfig legacyConfig = new ManagedLedgerConfig(); + legacyConfig.setMaxUnackedRangesToPersistInMetadataStore(-1); + legacyConfig.setThrottleMarkDelete(0); + ledger = factory.open(ledgerName, legacyConfig); + cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + assertThat(cursor.getMarkDeletedPosition()).isEqualTo(positions.get(6)); + assertThat(cursor.isMessageDeleted(positions.get(8))).isTrue(); + assertThat(cursor.isMessageDeleted(positions.get(7))).isFalse(); + assertThat(cursor.getProperties()).containsEntry("md-new", 6L); + assertThat(cursor.getCursorProperties()).containsEntry("cp-new", "cp-new-value"); + ledger.close(); + } + + @Test(timeOut = 30000) + public void testCheckpointCrossLedgerAckRef() throws Exception { + // Verifies that an AckStateRef pointing to an OLD cursor ledger (before rollover) + // is correctly fetched and applied during recovery. + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true); + config.setMaxEntriesPerLedger(3); // small msg-ledger → multiple msgLedgers + config.setMetadataMaxEntriesPerLedger(3); // force cursor-ledger rollover + config.setMaxUnackedRangesToPersistInMetadataStore(-1); + config.setThrottleMarkDelete(0); + + String ledgerName = "test_cross_ledger_ref"; + ManagedLedger ledger = factory.open(ledgerName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + + List positions = new ArrayList<>(); + for (int i = 0; i < 15; i++) { + positions.add(ledger.addEntry(("m-" + i).getBytes(Encoding))); + } + cursor.markDelete(positions.get(0)); + for (int i : new int[]{2, 5, 8, 11, 14}) { + cursor.delete(positions.get(i)); + } + // Advance mark-delete past the first msg-ledger to trigger rollovers + cursor.markDelete(positions.get(7)); + cursor.markDelete(positions.get(12)); + + Thread.sleep(500); // wait for async persists + rollovers + ledger.close(); + + // Reopen — recovery must fetch cross-ledger refs + ledger = factory.open(ledgerName, config); + cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + // All individual acks should be recovered, even those persisted in old cursor ledgers + for (int i : new int[]{2, 5, 8, 11, 14}) { + assertThat(cursor.isMessageDeleted(positions.get(i))).isTrue(); + } + ledger.close(); + } + + @Test(timeOut = 30000) + public void testCheckpointChunkedRecovery() throws Exception { + // Forces a single checkpoint to exceed maxEntrySize → chunked write + assembly. + // We can't easily produce a >5MB checkpoint with MockedBookKeeper, so we lower + // maxEntrySize to force chunking. + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true); + config.setPersistentUnackedRangesMaxEntrySize(1024); // very small → forces chunking + config.setMaxUnackedRangesToPersistInMetadataStore(-1); + config.setThrottleMarkDelete(0); + + String ledgerName = "test_chunked_recovery"; + ManagedLedger ledger = factory.open(ledgerName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + + List positions = new ArrayList<>(); + for (int i = 0; i < 200; i++) { + positions.add(ledger.addEntry(("m-" + i).getBytes(Encoding))); + } + cursor.markDelete(positions.get(0)); + // Many individual acks → large primary bitmap → checkpoint > 512 bytes → chunked + for (int i = 1; i < 200; i += 3) { + cursor.delete(positions.get(i)); + } + Thread.sleep(500); + ledger.close(); + + // Recovery must assemble chunks → parse checkpoint → apply state. + // Mark-delete position may have advanced beyond positions[0] due to individual + // ack side-effects, so we only assert on the recovered ack state. + ledger = factory.open(ledgerName, config); + cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + for (int i = 1; i < 200; i += 3) { + assertThat(cursor.isMessageDeleted(positions.get(i))).isTrue(); + } + ledger.close(); + } + + /** + * Regression: verifies that same-ledger individual acks (the most common case where + * the ack and mark-delete are in the same msgLedger) are correctly persisted and + * recovered. Before the markDirty fix, same-ledger acks were silently skipped by + * dirty tracking, resulting in empty checkpoints and data loss on recovery. + */ + @Test(timeOut = 30000) + public void testCheckpointSameLedgerAckSurvivesRecovery() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true); + config.setMaxUnackedRangesToPersistInMetadataStore(-1); + config.setThrottleMarkDelete(0); + + String ledgerName = "test_same_ledger_ack_recovery"; + ManagedLedger ledger = factory.open(ledgerName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + + // All entries in one msgLedger. md at entry 0, individual acks at entries 3, 7, 12. + List positions = new ArrayList<>(); + for (int i = 0; i < 20; i++) { + positions.add(ledger.addEntry(("m-" + i).getBytes(Encoding))); + } + cursor.markDelete(positions.get(0)); + cursor.delete(positions.get(3)); + cursor.delete(positions.get(7)); + cursor.delete(positions.get(12)); + + Thread.sleep(500); + ledger.close(); + + ledger = factory.open(ledgerName, config); + cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + assertThat(cursor.isMessageDeleted(positions.get(3))).isTrue(); + assertThat(cursor.isMessageDeleted(positions.get(7))).isTrue(); + assertThat(cursor.isMessageDeleted(positions.get(12))).isTrue(); + assertThat(cursor.isMessageDeleted(positions.get(1))).isFalse(); + ledger.close(); + } + + /** + * Verifies that recovery fails fast (not silent fallback) when a checkpoint + * references a ledger that can't be read. The cursor should rewind to ZK + * snapshot rather than silently dropping ack state. + */ + @Test(timeOut = 30000) + public void testCheckpointRefFetchFailureFailsRecovery() throws Exception { + TestPulsarMockBookKeeper bk = new TestPulsarMockBookKeeper(executor); + factory.shutdown(); + factory = new ManagedLedgerFactoryImpl(metadataStore, bk); + + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true); + config.setMaxEntriesPerLedger(3); + config.setMetadataMaxEntriesPerLedger(1); + config.setMaxUnackedRangesToPersistInMetadataStore(-1); + config.setThrottleMarkDelete(0); + + String ledgerName = "test_ref_fetch_fail"; + ManagedLedger ledger = factory.open(ledgerName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + + List positions = new ArrayList<>(); + for (int i = 0; i < 9; i++) { + positions.add(ledger.addEntry(("m-" + i).getBytes(Encoding))); + } + + cursor.markDelete(positions.get(0)); + cursor.delete(positions.get(2)); + cursor.delete(positions.get(5)); + Thread.sleep(500); + long refLedgerToBreak = cursor.getCursorLedger(); + cursor.markDelete(positions.get(3)); + Thread.sleep(500); + ledger.close(); + + bk.setErrorCodeMap(refLedgerToBreak, BKException.Code.BookieHandleNotAvailableException); + + ManagedLedgerFactoryImpl recoveryFactory = new ManagedLedgerFactoryImpl(metadataStore, bk); + ledger = recoveryFactory.open(ledgerName, config); + cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + assertThat(cursor.getMarkDeletedPosition()).isEqualTo(positions.get(3)); + assertThat(cursor.isMessageDeleted(positions.get(2))).isTrue(); + ledger.close(); + recoveryFactory.shutdown(); + } + + /** + * Verifies that if a checkpoint flush fails, the cursor keeps the previous + * successfully persisted state and the failed update is not recovered. + */ + @Test(timeOut = 30000) + public void testCheckpointFlushFailurePreservesPreviousState() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true); + config.setMaxEntriesPerLedger(3); + config.setMaxUnackedRangesToPersistInMetadataStore(-1); + config.setThrottleMarkDelete(0); + + String ledgerName = "test_checkpoint_flush_fail"; + ManagedLedger ledger = factory.open(ledgerName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + + List positions = new ArrayList<>(); + for (int i = 0; i < 12; i++) { + positions.add(ledger.addEntry(("f-" + i).getBytes(Encoding))); + } + + cursor.markDelete(positions.get(0)); + cursor.delete(positions.get(2)); + cursor.delete(positions.get(5)); + final ManagedCursorImpl persistedCursor = cursor; + Awaitility.await().untilAsserted( + () -> assertThat(persistedCursor.getStats().getPersistLedgerSucceed()).isGreaterThan(0)); + + bkc.addEntryFailAfter(0, BKException.Code.NoBookieAvailableException); + bkc.addEntryFailAfter(1, BKException.Code.NoBookieAvailableException); + cursor.delete(positions.get(8)); + final ManagedCursorImpl failedCursor = cursor; + Awaitility.await().untilAsserted( + () -> assertThat(failedCursor.getStats().getPersistLedgerErrors()).isGreaterThan(0)); + + ledger.close(); + + ManagedLedgerFactoryImpl recoveryFactory = new ManagedLedgerFactoryImpl(metadataStore, bkc); + ManagedLedger recoveredLedger = recoveryFactory.open(ledgerName, config); + ManagedCursorImpl recoveredCursor = (ManagedCursorImpl) recoveredLedger.openCursor("c1"); + assertThat(recoveredCursor.isMessageDeleted(positions.get(2))).isTrue(); + assertThat(recoveredCursor.isMessageDeleted(positions.get(5))).isTrue(); + recoveredLedger.close(); + recoveryFactory.shutdown(); + } + + /** + * Regression: after recovery the per-ledger {@code lastCheckpointPos} bookkeeping must be rebuilt + * from the recovered checkpoint. Without it, the first persist after a restart with multiple + * active msg ledgers throws "Missing lastCheckpointPos" and silently falls back to the metadata + * store, defeating per-msgLedger persistence. + */ + @Test(timeOut = 30000) + public void testCheckpointRecoveryRebuildsLastAppendedPos() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true); + config.setMaxEntriesPerLedger(5); + config.setMaxUnackedRangesToPersistInMetadataStore(-1); + config.setThrottleMarkDelete(0); + + String ledgerName = "test_checkpoint_rebuild_last_appended_pos"; + ManagedLedger ledger = factory.open(ledgerName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + List positions = new ArrayList<>(); + for (int i = 0; i < 20; i++) { + positions.add(ledger.addEntry(("msg-" + i).getBytes(Encoding))); + } + cursor.markDelete(positions.get(0)); + cursor.delete(positions.get(3)); + cursor.delete(positions.get(7)); + cursor.delete(positions.get(12)); + cursor.delete(positions.get(18)); + Thread.sleep(500); + ledger.close(); + + // Reopen, then keep using the cursor — the first post-recovery persist must succeed via the + // cursor ledger instead of silently degrading to the metadata store. + ledger = factory.open(ledgerName, config); + cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + assertThat(cursor.isMessageDeleted(positions.get(3))).isTrue(); + assertThat(cursor.isMessageDeleted(positions.get(12))).isTrue(); + + cursor.delete(positions.get(5)); + cursor.markDelete(positions.get(10)); + ManagedCursorImpl cursorRef = cursor; + Awaitility.await().untilAsserted( + () -> assertThat(cursorRef.getStats().getPersistLedgerSucceed()).isGreaterThan(0)); + ledger.close(); + } + + /** + * Regression: an AckStateRef points at the commit entry of a checkpoint; for a chunked + * checkpoint that is the last chunk part. Ref recovery must assemble the chunk before + * extracting the ack state, otherwise a cursor with many active msg ledgers and a small + * maxEntrySize fails recovery and rewinds to the ZK snapshot, losing individual acks. + */ + @Test(timeOut = 60000) + public void testCheckpointChunkedRefRecovery() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true); + config.setPersistentUnackedRangesMaxEntrySize(1024); // small → per-ledger checkpoints chunk + config.setMaxEntriesPerLedger(3); // many msg ledgers → many refs per checkpoint + config.setMaxUnackedRangesToPersistInMetadataStore(-1); + config.setThrottleMarkDelete(0); + + String ledgerName = "test_checkpoint_chunked_ref_recovery"; + ManagedLedger ledger = factory.open(ledgerName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + List positions = new ArrayList<>(); + for (int i = 0; i < 150; i++) { + positions.add(ledger.addEntry(("m-" + i).getBytes(Encoding))); + } + cursor.markDelete(positions.get(0)); + // One ack in nearly every msg ledger → each per-ledger checkpoint carries ~50 refs, + // pushing it over maxEntrySize and forcing chunked writes. + for (int i = 1; i < 150; i += 3) { + cursor.delete(positions.get(i)); + } + Thread.sleep(1000); + ledger.close(); + + ledger = factory.open(ledgerName, config); + cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + for (int i = 1; i < 150; i += 3) { + assertThat(cursor.isMessageDeleted(positions.get(i))) + .as("chunked ref recovery should preserve ack at " + i) + .isTrue(); + } + ledger.close(); + } + + /** + * Regression: when a flush fails, only the ledgers whose checkpoints were not appended are + * re-marked dirty; the next flush must retry them — including a ledger that has never been + * persisted before. Without this, a first-time dirty ledger whose append failed would have no + * lastCheckpointPos and no dirty flag, wedging every subsequent persist into "Missing + * lastCheckpointPos" and permanently degrading to the metadata store. + */ + @Test(timeOut = 30000) + public void testCheckpointFailedFlushRetriesMissingLedgers() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true); + config.setMaxEntriesPerLedger(3); + config.setMaxUnackedRangesToPersistInMetadataStore(-1); + config.setThrottleMarkDelete(0); + + String ledgerName = "test_checkpoint_retry_failed_ledgers"; + ManagedLedger ledger = factory.open(ledgerName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + List positions = new ArrayList<>(); + for (int i = 0; i < 20; i++) { + positions.add(ledger.addEntry(("m-" + i).getBytes(Encoding))); + } + + // Establish persisted positions for the first two msg ledgers (2 successful flushes). + cursor.markDelete(positions.get(0)); + cursor.delete(positions.get(5)); // gap from mark-delete → no implicit flush + cursor.markDelete(positions.get(1)); + ManagedCursorImpl persistedCursor = cursor; + Awaitility.await().untilAsserted( + () -> assertThat(persistedCursor.getStats().getPersistLedgerSucceed()).isGreaterThanOrEqualTo(2)); + + // Fail the next flush entirely: a first-time dirty ledger (5) plus the mark-delete ledger. + bkc.addEntryFailAfter(0, BKException.Code.NoBookieAvailableException); + bkc.addEntryFailAfter(1, BKException.Code.NoBookieAvailableException); + cursor.delete(positions.get(16)); // msg ledger 5, never persisted before + cursor.markDelete(positions.get(3)); + ManagedCursorImpl failedCursor = cursor; + Awaitility.await().untilAsserted( + () -> assertThat(failedCursor.getStats().getPersistLedgerErrors()).isGreaterThan(0)); + + // The next flush (mark-delete in a different ledger) must retry msg ledger 5. + cursor.markDelete(positions.get(9)); + ManagedCursorImpl retriedCursor = cursor; + Awaitility.await().untilAsserted( + () -> assertThat(retriedCursor.getStats().getPersistLedgerSucceed()).isGreaterThan(2)); + ledger.close(); + + // The retried ack must survive recovery. + ledger = factory.open(ledgerName, config); + cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + assertThat(cursor.isMessageDeleted(positions.get(16))).isTrue(); + assertThat(cursor.isMessageDeleted(positions.get(14))).isFalse(); + ledger.close(); + } + + /** + * Regression: the first flush must succeed even when the mark-delete ledger itself holds + * individual acks that have never been persisted (e.g. the first acks arrived non-contiguously + * and did not trigger an implicit mark-delete). The mark-delete ledger's checkpoint is written + * first so other checkpoints in the same flush can reference it. + */ + @Test(timeOut = 30000) + public void testCheckpointFirstFlushWithMdLedgerAcks() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true); + config.setMaxEntriesPerLedger(3); + config.setMaxUnackedRangesToPersistInMetadataStore(-1); + config.setThrottleMarkDelete(0); + + String ledgerName = "test_checkpoint_first_flush_md_ledger_acks"; + ManagedLedger ledger = factory.open(ledgerName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + List positions = new ArrayList<>(); + for (int i = 0; i < 20; i++) { + positions.add(ledger.addEntry(("m-" + i).getBytes(Encoding))); + } + + // Non-contiguous acks: one in the mark-delete ledger (0), one in a later ledger (1). + // Neither triggers an implicit mark-delete, so this is the first ledger persist. + cursor.delete(positions.get(2)); + cursor.delete(positions.get(5)); + cursor.markDelete(positions.get(1)); + + ManagedCursorImpl flushedCursor = cursor; + Awaitility.await().untilAsserted( + () -> assertThat(flushedCursor.getStats().getPersistLedgerSucceed()).isGreaterThan(0)); + ledger.close(); + + ledger = factory.open(ledgerName, config); + cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + assertThat(cursor.isMessageDeleted(positions.get(2))).isTrue(); + assertThat(cursor.isMessageDeleted(positions.get(5))).isTrue(); + assertThat(cursor.isMessageDeleted(positions.get(3))).isFalse(); + ledger.close(); + } + + /** + * Regression: a batch-index ack in a msg ledger that has no individual acks must still be + * persisted and survive recovery. Batch acks are tracked in {@code batchDeletedIndexes} only; + * without dirty-marking and ref enumeration for those ledgers, the batch ack is written once + * and then becomes unreachable from later checkpoints, so it is lost across restart. + */ + @Test(timeOut = 30000) + public void testCheckpointBatchAckInNonMdLedgerSurvivesRecovery() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true); + config.setMaxEntriesPerLedger(3); + config.setMaxUnackedRangesToPersistInMetadataStore(-1); + config.setMaxBatchDeletedIndexToPersist(1000); + config.setDeletionAtBatchIndexLevelEnabled(true); + config.setThrottleMarkDelete(0); + + String ledgerName = "test_checkpoint_batch_ack_non_md_ledger"; + ManagedLedger ledger = factory.open(ledgerName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + List positions = new ArrayList<>(); + for (int i = 0; i < 9; i++) { + positions.add(ledger.addEntry(("entry-" + i).getBytes(Encoding))); + } + cursor.markDelete(positions.get(1)); // md ledger 0 + + // Batch ack in msg ledger 2 (entry 7): no individual ack range, so dirty-marking must + // come from the batch-index path. + Position batchPos = positions.get(7); + BitSet bitSet = new BitSet(32); + bitSet.set(2, 5); + cursor.batchDeletedIndexes.put(batchPos, bitSet); + + cursor.markDelete(positions.get(4)); // flush A: writes ledger 2 (dirty) + md ledger 1 + cursor.markDelete(positions.get(5)); // flush B: md ledger 1 must ref ledger 2 + ManagedCursorImpl flushedCursor = cursor; + Awaitility.await().untilAsserted( + () -> assertThat(flushedCursor.getStats().getPersistLedgerSucceed()).isGreaterThanOrEqualTo(2)); + ledger.close(); + + ledger = factory.open(ledgerName, config); + cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + assertThat(cursor.getDeletedBatchIndexesAsLongArray(positions.get(7))).isNotEmpty(); + ledger.close(); + } + + /** + * Regression: old cursor ledgers that are only referenced via AckStateRefs (cross-ledger refs + * from before a restart) must be tracked after recovery and reclaimed by GC once mark-delete + * passes the last msg ledger holding acks. Without recovery-time tracking, those ledgers leak + * forever across restarts. + */ + @Test(timeOut = 60000) + public void testCheckpointGcReclaimsRefLedgersAcrossRestart() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true); + config.setMaxEntriesPerLedger(3); + config.setMetadataMaxEntriesPerLedger(6); // exactly one rollover in phase 1 + config.setMaxUnackedRangesToPersistInMetadataStore(-1); + config.setThrottleMarkDelete(0); + + String ledgerName = "test_checkpoint_gc_across_restart"; + ManagedLedger ledger = factory.open(ledgerName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + List positions = new ArrayList<>(); + for (int i = 0; i < 15; i++) { + positions.add(ledger.addEntry(("m-" + i).getBytes(Encoding))); + } + cursor.markDelete(positions.get(0)); + long firstCursorLedger = cursor.getCursorLedger(); + for (int i : new int[]{5, 8, 11, 14}) { // gaps → no implicit mark-delete + cursor.delete(positions.get(i)); + } + cursor.markDelete(positions.get(1)); // flush: 5 checkpoints in the first cursor ledger + // The next mark-delete pushes LAC to 6 and rolls the cursor ledger over; it writes + // the first checkpoint into the new ledger, with refs pointing at the old one. + cursor.markDelete(positions.get(7)); + long secondCursorLedger = cursor.getCursorLedger(); + assertThat(secondCursorLedger).isNotEqualTo(firstCursorLedger); + ManagedCursorImpl cursorBeforeClose = cursor; + Awaitility.await().untilAsserted( + () -> assertThat(cursorBeforeClose.getStats().getPersistLedgerSucceed()).isGreaterThanOrEqualTo(3)); + ledger.close(); + + // Restart: refs into the pre-restart cursor ledger must be tracked for GC. + ledger = factory.open(ledgerName, config); + cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + assertThat(cursor.isMessageDeleted(positions.get(11))).isTrue(); + assertThat(cursor.isMessageDeleted(positions.get(14))).isTrue(); + + // Absorb all remaining acks, then force rollovers so GC runs. + for (int i = 15; i < 35; i++) { + positions.add(ledger.addEntry(("m-" + i).getBytes(Encoding))); + } + cursor.markDelete(positions.get(19)); // first persist creates a fresh cursor ledger + cursor.markDelete(positions.get(23)); // absorbs remaining acks + cursor.markDelete(positions.get(25)); + cursor.markDelete(positions.get(27)); + cursor.markDelete(positions.get(29)); + cursor.markDelete(positions.get(31)); // forces the next rollover → GC + + Set expectedReclaimed = new HashSet<>(); + expectedReclaimed.add(firstCursorLedger); + expectedReclaimed.add(secondCursorLedger); + Awaitility.await().untilAsserted(() -> { + Set live = bkc.getLedgers(); + assertThat(live).doesNotContainAnyElementsOf(expectedReclaimed); + }); + ledger.close(); + } + + /** + * Unlike the legacy single-PositionInfo path (which truncates batch indexes at + * {@code maxBatchDeletedIndexToPersist}), the per-ledger path does NOT truncate: each + * checkpoint is bounded by chunking, so all batch indexes survive recovery. + */ + @Test(timeOut = 30000) + public void testCheckpointBatchAckNoTruncationInPerLedgerMode() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true); + config.setMaxEntriesPerLedger(5); + config.setMaxBatchDeletedIndexToPersist(3); + config.setDeletionAtBatchIndexLevelEnabled(true); + config.setMaxUnackedRangesToPersistInMetadataStore(-1); + config.setThrottleMarkDelete(0); + + // Topic-format name: the legacy close-time meta-store snapshot still applies batch-index + // truncation, whose OTEL metric builds attributes via TopicName parsing. + String ledgerName = "my-tenant/my-ns/persistent/test_checkpoint_batch_ack_no_truncation"; + ManagedLedger ledger = factory.open(ledgerName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + List positions = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + positions.add(ledger.addEntry(("entry-" + i).getBytes(Encoding))); + } + cursor.markDelete(positions.get(0)); + // Five batch acks in the same msg ledger (entries 5-9, ledger 4). Per-ledger checkpoints are + // bounded by chunking, so batch indexes are NOT truncated here (unlike the legacy single + // PositionInfo path, which applies maxBatchDeletedIndexToPersist). + for (int i = 5; i <= 9; i++) { + BitSet bitSet = new BitSet(32); + bitSet.set(2, 4); + cursor.batchDeletedIndexes.put(positions.get(i), bitSet); + } + cursor.markDelete(positions.get(2)); // flush + ManagedCursorImpl flushedCursor = cursor; + Awaitility.await().untilAsserted( + () -> assertThat(flushedCursor.getStats().getPersistLedgerSucceed()).isGreaterThan(0)); + ledger.close(); + + ledger = factory.open(ledgerName, config); + cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + for (int i = 5; i <= 9; i++) { + assertThat(cursor.getDeletedBatchIndexesAsLongArray(positions.get(i))) + .as("batch ack at position " + i).isNotEmpty(); + } + ledger.close(); + } + + /** + * A per-msgLedger checkpoint write failure must not fall back to the metadata store: the unpersisted update stays + * in memory, is retried on the next mark-delete, and ZK is only touched by the infrequent + * rollover bookkeeping. This keeps the failure path off ZK (which is the point of per-ledger + * persistence) and relies on at-least-once redelivery as the safety net. + */ + @Test(timeOut = 30000) + public void testCheckpointPersistFailureSkipsMetaStoreFallback() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true); + config.setMaxEntriesPerLedger(3); + config.setMaxUnackedRangesToPersistInMetadataStore(-1); + config.setThrottleMarkDelete(0); + + String ledgerName = "test_checkpoint_no_zk_fallback"; + ManagedLedger ledger = factory.open(ledgerName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + List positions = new ArrayList<>(); + for (int i = 0; i < 12; i++) { + positions.add(ledger.addEntry(("m-" + i).getBytes(Encoding))); + } + cursor.markDelete(positions.get(0)); + cursor.delete(positions.get(5)); // gaps → no implicit mark-delete + cursor.delete(positions.get(8)); + cursor.markDelete(positions.get(1)); // successful flush with acks 5, 8 + ManagedCursorImpl persistedCursor = cursor; + Awaitility.await().untilAsserted( + () -> assertThat(persistedCursor.getStats().getPersistLedgerSucceed()).isGreaterThanOrEqualTo(2)); + long zkBefore = cursor.getStats().getPersistZookeeperSucceed(); + + // Fail the next flush entirely (ack 11 in msg ledger 3 + mark-delete ledger). + bkc.addEntryFailAfter(0, BKException.Code.NoBookieAvailableException); + bkc.addEntryFailAfter(1, BKException.Code.NoBookieAvailableException); + cursor.delete(positions.get(11)); + cursor.markDelete(positions.get(4)); + ManagedCursorImpl failedCursor = cursor; + Awaitility.await().untilAsserted( + () -> assertThat(failedCursor.getStats().getPersistLedgerErrors()).isGreaterThan(0)); + // The failure must not have written the metadata store. + assertThat(cursor.getStats().getPersistZookeeperSucceed()).isEqualTo(zkBefore); + + // The next mark-delete retries on a fresh cursor ledger and persists the failed ack. + cursor.markDelete(positions.get(7)); + ManagedCursorImpl retriedCursor = cursor; + Awaitility.await().untilAsserted( + () -> assertThat(retriedCursor.getStats().getPersistLedgerSucceed()).isGreaterThan(2)); + ledger.close(); + + ledger = factory.open(ledgerName, config); + cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + assertThat(cursor.isMessageDeleted(positions.get(11))).isTrue(); + assertThat(cursor.isMessageDeleted(positions.get(10))).isFalse(); + ledger.close(); + } + + /** + * A resetCursor whose own mark-delete persist fails must still apply its in-memory state change + * (deferred-persist semantics): the reset completes, the ack state is cleared in memory, and the + * next flush persists the cleared state. This also guards the reset-vs-failed-ack race: a failed + * normal mark-delete must never run the default align (which would advance + * persistentMarkDeletePosition as if the write had succeeded). + */ + @Test(timeOut = 30000) + public void testCheckpointResetCursorSurvivesPersistFailure() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true); + config.setMaxEntriesPerLedger(3); + config.setMaxUnackedRangesToPersistInMetadataStore(-1); + config.setThrottleMarkDelete(0); + + String ledgerName = "test_checkpoint_reset_persist_failure"; + ManagedLedger ledger = factory.open(ledgerName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + List positions = new ArrayList<>(); + for (int i = 0; i < 9; i++) { + positions.add(ledger.addEntry(("m-" + i).getBytes(Encoding))); + } + cursor.markDelete(positions.get(0)); + cursor.delete(positions.get(5)); // gaps → no implicit mark-delete + cursor.delete(positions.get(8)); + cursor.markDelete(positions.get(1)); // persists acks 5, 8 + ManagedCursorImpl persistedCursor = cursor; + Awaitility.await().untilAsserted( + () -> assertThat(persistedCursor.getStats().getPersistLedgerSucceed()).isGreaterThanOrEqualTo(2)); + + // Fail the reset's own ledger persist (both checkpoints in the flush). + // BK writes fail; the reset falls back to the md-only tombstone in ZK (the post-reset + // state has no holes, so the ZK form is complete). + bkc.addEntryFailAfter(0, BKException.Code.NoBookieAvailableException); + bkc.addEntryFailAfter(1, BKException.Code.NoBookieAvailableException); + cursor.resetCursor(positions.get(0)); + // The reset succeeded durably (via ZK) and only then applied in memory. + assertThat(cursor.isMessageDeleted(positions.get(5))).isFalse(); + assertThat(cursor.isMessageDeleted(positions.get(8))).isFalse(); + ledger.close(); + + // Recovery reads the ZK tombstone: pre-reset acks must not resurrect. + ledger = factory.open(ledgerName, config); + cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + assertThat(cursor.getMarkDeletedPosition()).isEqualTo(ledger.getPreviousPosition(positions.get(0))); + assertThat(cursor.isMessageDeleted(positions.get(5))).isFalse(); + assertThat(cursor.isMessageDeleted(positions.get(8))).isFalse(); + ledger.close(); + } + + @Test(timeOut = 30000) + public void testCheckpointResetCursorCleanPersistRecovery() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true); + config.setMaxEntriesPerLedger(3); + config.setMaxUnackedRangesToPersistInMetadataStore(-1); + config.setThrottleMarkDelete(0); + + String ledgerName = "test_checkpoint_reset_clean_recovery"; + ManagedLedger ledger = factory.open(ledgerName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + List positions = new ArrayList<>(); + for (int i = 0; i < 9; i++) { + positions.add(ledger.addEntry(("m-" + i).getBytes(Encoding))); + } + cursor.markDelete(positions.get(0)); + cursor.delete(positions.get(5)); + cursor.delete(positions.get(8)); + cursor.markDelete(positions.get(1)); // persists acks 5, 8 + ManagedCursorImpl persistedCursor = cursor; + Awaitility.await().untilAsserted( + () -> assertThat(persistedCursor.getStats().getPersistLedgerSucceed()).isGreaterThanOrEqualTo(2)); + + // Clean-path reset: the tombstone checkpoint (empty ack state, new mark-delete) is + // persisted without failures. + cursor.resetCursor(positions.get(0)); + assertThat(cursor.isMessageDeleted(positions.get(5))).isFalse(); + ManagedCursorImpl resetCursor = cursor; + Awaitility.await().untilAsserted( + () -> assertThat(resetCursor.getStats().getPersistLedgerSucceed()).isGreaterThan(2)); + ledger.close(); + + // Recovery takes the tombstone: pre-reset acks must not resurrect. + ledger = factory.open(ledgerName, config); + cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + assertThat(cursor.getMarkDeletedPosition()).isEqualTo(ledger.getPreviousPosition(positions.get(0))); + assertThat(cursor.isMessageDeleted(positions.get(5))).isFalse(); + assertThat(cursor.isMessageDeleted(positions.get(8))).isFalse(); + ledger.close(); + } + + @Test(timeOut = 30000) + public void testCheckpointResetConsistentWhenAllPersistsFail() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true); + config.setMaxEntriesPerLedger(3); + config.setMaxUnackedRangesToPersistInMetadataStore(-1); + config.setThrottleMarkDelete(0); + + String ledgerName = "test_checkpoint_reset_lost_before_tombstone"; + ManagedLedger ledger = factory.open(ledgerName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + List positions = new ArrayList<>(); + for (int i = 0; i < 9; i++) { + positions.add(ledger.addEntry(("m-" + i).getBytes(Encoding))); + } + cursor.markDelete(positions.get(0)); + cursor.delete(positions.get(5)); + cursor.delete(positions.get(8)); + cursor.markDelete(positions.get(1)); // persists acks 5, 8 + ManagedCursorImpl persistedCursor = cursor; + Awaitility.await().untilAsserted( + () -> assertThat(persistedCursor.getStats().getPersistLedgerSucceed()).isGreaterThanOrEqualTo(2)); + + // Make every subsequent BK append fail AND the ZK cursor-info write fail: neither + // tombstone destination is available, so the reset must fail visibly. + for (int i = 0; i < 100; i++) { + bkc.addEntryFailAfter(i, BKException.Code.NoBookieAvailableException); + } + metadataStore.failConditional(new MetadataStoreException("stop zk writes"), + (operationType, path) -> operationType == FaultInjectionMetadataStore.OperationType.PUT + && path.equals("/managed-ledgers/" + ledgerName + "/c1")); + try { + cursor.resetCursor(positions.get(0)); + fail("reset should fail when both BK and ZK writes fail"); + } catch (Exception e) { + // resetFailed is expected: no durable tombstone could be written. + } + // Consistency-first: the in-memory state is NOT reset before the tombstone is durable. + assertThat(cursor.isMessageDeleted(positions.get(5))).isTrue(); + assertThat(cursor.isMessageDeleted(positions.get(8))).isTrue(); + + // Simulate a broker crash + restart at this point: a second factory recovers purely + // from the persisted log, and sees exactly what the first instance still holds in + // memory. No silent reset loss window. + ManagedLedgerFactoryImpl factory2 = new ManagedLedgerFactoryImpl(metadataStore, bkc); + ManagedLedger ledger2 = factory2.open(ledgerName, config); + ManagedCursorImpl recovered = (ManagedCursorImpl) ledger2.openCursor("c1"); + assertThat(recovered.isMessageDeleted(positions.get(5))).isTrue(); + assertThat(recovered.isMessageDeleted(positions.get(8))).isTrue(); + factory2.shutdownAsync().get(10, TimeUnit.SECONDS); + } + + @Test(timeOut = 30000) + public void testCheckpointClearBacklogRecovery() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true); + config.setMaxEntriesPerLedger(3); + config.setMaxUnackedRangesToPersistInMetadataStore(-1); + config.setThrottleMarkDelete(0); + + String ledgerName = "test_checkpoint_clear_backlog_recovery"; + ManagedLedger ledger = factory.open(ledgerName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + List positions = new ArrayList<>(); + for (int i = 0; i < 9; i++) { + positions.add(ledger.addEntry(("m-" + i).getBytes(Encoding))); + } + cursor.markDelete(positions.get(0)); + cursor.delete(positions.get(5)); // hole survives: no contiguous mark-delete + cursor.markDelete(positions.get(1)); + ManagedCursorImpl persistedCursor = cursor; + Awaitility.await().untilAsserted( + () -> assertThat(persistedCursor.getStats().getPersistLedgerSucceed()).isGreaterThanOrEqualTo(2)); + assertThat(cursor.isMessageDeleted(positions.get(5))).isTrue(); + + // clearBacklog absorbs every hole via mark-delete-to-tail and invalidates refs. + CountDownLatch cleared = new CountDownLatch(1); + cursor.asyncClearBacklog(new AsyncCallbacks.ClearBacklogCallback() { + @Override + public void clearBacklogComplete(Object ctx) { + cleared.countDown(); + } + + @Override + public void clearBacklogFailed(ManagedLedgerException exception, Object ctx) { + // no-op + } + }, null); + cleared.await(10, TimeUnit.SECONDS); + assertThat(cursor.isMessageDeleted(positions.get(5))).isTrue(); // absorbed, not a hole + ledger.close(); + + ledger = factory.open(ledgerName, config); + cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + // After recovery the backlog is drained: mark-delete at tail, no individual holes. + assertThat(cursor.getMarkDeletedPosition()).isEqualTo(positions.get(8)); + assertThat(cursor.isMessageDeleted(positions.get(5))).isTrue(); + ledger.close(); + } + + @Test(timeOut = 30000) + public void testCursorStatsIndividualDeletedMessages() throws Exception { + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setMaxEntriesPerLedger(5); + config.setThrottleMarkDelete(0); + + String ledgerName = "test_cursor_stats_individual_deleted"; + ManagedLedger ledger = factory.open(ledgerName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1"); + List positions = new ArrayList<>(); + for (int i = 0; i < 15; i++) { + positions.add(ledger.addEntry(("m-" + i).getBytes(Encoding))); + } + + cursor.markDelete(positions.get(0)); + + // No holes yet. + ManagedLedgerInternalStats.CursorStats stats = cursor.getCursorStats(); + assertThat(stats.individualDeletedMessagesCount).isEqualTo(0); + assertThat(stats.firstIndividualDeletedMessage).isNull(); + + // Contiguous holes 2,3,4 (merge into one range) plus isolated holes at 7 and 12. + cursor.delete(positions.get(2)); + cursor.delete(positions.get(3)); + cursor.delete(positions.get(4)); + cursor.delete(positions.get(7)); + cursor.delete(positions.get(12)); + + stats = cursor.getCursorStats(); + assertThat(stats.individualDeletedMessagesCount).isEqualTo(5); + // First hole is positions.get(2) — the start of the contiguous run, not its end (4). + assertThat(stats.firstIndividualDeletedMessage).isEqualTo(positions.get(2).toString()); + + // Mark-delete past the contiguous run: the first hole becomes 7. + cursor.markDelete(positions.get(4)); + stats = cursor.getCursorStats(); + assertThat(stats.individualDeletedMessagesCount).isEqualTo(2); + assertThat(stats.firstIndividualDeletedMessage).isEqualTo(positions.get(7).toString()); + + // Mark-delete past all holes: count drops to 0. + cursor.markDelete(positions.get(13)); + stats = cursor.getCursorStats(); + assertThat(stats.individualDeletedMessagesCount).isEqualTo(0); + assertThat(stats.firstIndividualDeletedMessage).isNull(); + + ledger.close(); + } + } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetCompatibilityTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetCompatibilityTest.java index 474ddd41e5c21..75e01555a94ba 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetCompatibilityTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetCompatibilityTest.java @@ -33,8 +33,8 @@ import org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.PositionFactory; -import org.apache.bookkeeper.mledger.proto.MLDataFormats; import org.apache.bookkeeper.mledger.proto.MLDataFormats.MessageRange; +import org.apache.bookkeeper.mledger.proto.MLDataFormats.NestedPositionInfo; import org.apache.bookkeeper.test.BookKeeperClusterTestCase; import org.apache.pulsar.common.util.collections.LongPairRangeSet; import org.apache.pulsar.common.util.collections.OpenLongPairRangeSet; @@ -180,9 +180,9 @@ public void testBitmapBinaryFormat() { private MessageRange createMessageRange(long lowerLedger, long lowerEntry, long upperLedger, long upperEntry) { return MessageRange.newBuilder() - .setLowerEndpoint(MLDataFormats.NestedPositionInfo.newBuilder() + .setLowerEndpoint(NestedPositionInfo.newBuilder() .setLedgerId(lowerLedger).setEntryId(lowerEntry)) - .setUpperEndpoint(MLDataFormats.NestedPositionInfo.newBuilder() + .setUpperEndpoint(NestedPositionInfo.newBuilder() .setLedgerId(upperLedger).setEntryId(upperEntry)) .build(); } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetDirtyTrackingTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetDirtyTrackingTest.java new file mode 100644 index 0000000000000..b3fc46b61f424 --- /dev/null +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetDirtyTrackingTest.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.impl; + +import static org.assertj.core.api.Assertions.assertThat; +import java.util.Set; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.pulsar.common.util.collections.LongPairRangeSet.LongPairConsumer; +import org.testng.annotations.Test; + +/** + * Regression tests for {@link PositionRangeSet} dirty tracking. + * + *

These tests cover the three markDirty fixes: + *

    + *
  1. Same-ledger acks (upperLedgerId == lowerLedgerId) now mark dirty — original code + * had {@code upperLedgerId <= lowerLedgerId} which silently skipped them.
    2. + *
  2. Range end is inclusive — original {@code add(L+1, U+1)} missed the upper ledger; + * fixed to {@code add(L+1, U+2)}.
    3. + *
  3. snapshotAndClearDirtyLedgers correctly maps dirty bits back to raw ledger IDs.
    4. + *
+ */ +public class PositionRangeSetDirtyTrackingTest { + + private static final LongPairConsumer CONVERTER = + (ledgerId, entryId) -> PositionFactory.create(ledgerId, entryId); + + @Test + public void testSameLedgerAckMarksDirty() { + // Regression: same-ledger ack (the most common case) was silently skipped by + // the original markDirty when upperLedgerId == lowerLedgerId. + PositionRangeSet set = new PositionRangeSet(CONVERTER, true); + set.addOpenClosed(100, 0, 100, 5); + Set dirty = set.snapshotAndClearDirtyLedgers(); + assertThat(dirty).containsExactly(100L); + } + + @Test + public void testCrossLedgerRangeMarksAllInclusive() { + // Regression: markDirty(L1, L2) should mark L1 through L2 INCLUSIVE. + // Original add(L1+1, L2+1) [half-open] missed L2. + PositionRangeSet set = new PositionRangeSet(CONVERTER, true); + set.addOpenClosed(100, 0, 103, 0); + Set dirty = set.snapshotAndClearDirtyLedgers(); + assertThat(dirty).containsExactlyInAnyOrder(100L, 101L, 102L, 103L); + } + + @Test + public void testDirtyClearedAfterSnapshot() { + PositionRangeSet set = new PositionRangeSet(CONVERTER, true); + set.addOpenClosed(100, 0, 100, 5); + set.snapshotAndClearDirtyLedgers(); + // After snapshot, dirty should be empty + Set dirty2 = set.snapshotAndClearDirtyLedgers(); + assertThat(dirty2).isEmpty(); + } + + @Test + public void testRestoreDirtyLedgers() { + PositionRangeSet set = new PositionRangeSet(CONVERTER, true); + set.addOpenClosed(100, 0, 100, 5); + Set dirty = set.snapshotAndClearDirtyLedgers(); + // Restore dirty ledgers (simulates failed persist) + set.restoreDirtyLedgers(dirty); + // Next snapshot should have them again + Set dirty2 = set.snapshotAndClearDirtyLedgers(); + assertThat(dirty2).containsExactly(100L); + } + + @Test + public void testMultipleLedgersIndividuallyDirty() { + PositionRangeSet set = new PositionRangeSet(CONVERTER, true); + set.addOpenClosed(100, 0, 100, 3); + set.addOpenClosed(200, 0, 200, 7); + set.addOpenClosed(300, 0, 300, 1); + Set dirty = set.snapshotAndClearDirtyLedgers(); + assertThat(dirty).containsExactlyInAnyOrder(100L, 200L, 300L); + } + + @Test + public void testBitmapOfReturnsNullForEmptyLedger() { + PositionRangeSet set = new PositionRangeSet(CONVERTER, true); + assertThat(set.bitmapOf(999)).isNull(); + } + + @Test + public void testBitmapOfReturnsBytesForActiveLedger() { + PositionRangeSet set = new PositionRangeSet(CONVERTER, true); + set.addOpenClosed(100, 0, 100, 5); + byte[] bytes = set.bitmapOf(100); + assertThat(bytes).isNotNull(); + assertThat(bytes.length).isGreaterThan(0); + } + + @Test + public void testForEachActiveLedger() { + PositionRangeSet set = new PositionRangeSet(CONVERTER, true); + set.addOpenClosed(100, 0, 100, 1); + set.addOpenClosed(200, 0, 200, 1); + Set active = new java.util.HashSet<>(); + set.forEachActiveLedger(active::add); + assertThat(active).containsExactlyInAnyOrder(100L, 200L); + } + + @Test + public void testDirtyDisabledWhenMultiEntryOff() { + // When enableMultiEntry is false, dirty tracking is a no-op. + PositionRangeSet set = new PositionRangeSet(CONVERTER, false); + set.addOpenClosed(100, 0, 100, 5); + Set dirty = set.snapshotAndClearDirtyLedgers(); + assertThat(dirty).isEmpty(); + } + + @Test + public void testSameLedgerMarkDirty() { + PositionRangeSet set = new PositionRangeSet(CONVERTER, true); + set.addOpenClosed(0, -1, 0, 0); + Set dirty = set.snapshotAndClearDirtyLedgers(); + assertThat(dirty).containsExactly(0L); + } + + @Test + public void testCrossLedgerMarkDirtyUpperBound() { + PositionRangeSet set = new PositionRangeSet(CONVERTER, true); + set.addOpenClosed(100, 0, 103, 0); + Set dirty = set.snapshotAndClearDirtyLedgers(); + assertThat(dirty).containsExactlyInAnyOrder(100L, 101L, 102L, 103L); + } + + @Test + public void testRestoreAfterSnapshot() { + PositionRangeSet set = new PositionRangeSet(CONVERTER, true); + set.addOpenClosed(50, 0, 50, 1); + set.addOpenClosed(60, 0, 60, 1); + Set dirty = set.snapshotAndClearDirtyLedgers(); + assertThat(dirty).containsExactlyInAnyOrder(50L, 60L); + assertThat(set.snapshotAndClearDirtyLedgers()).isEmpty(); + set.restoreDirtyLedgers(dirty); + assertThat(set.snapshotAndClearDirtyLedgers()).containsExactlyInAnyOrder(50L, 60L); + } + + @Test + public void testRemoveAtMostClearsDirty() { + PositionRangeSet set = new PositionRangeSet(CONVERTER, true); + set.addOpenClosed(10, 0, 20, 0); + set.removeAtMost(15, 0); + Set dirty = set.snapshotAndClearDirtyLedgers(); + for (long id : dirty) { + assertThat(id).isGreaterThan(14); + } + } + + @Test + public void testEmptySnapshot() { + PositionRangeSet set = new PositionRangeSet(CONVERTER, true); + assertThat(set.snapshotAndClearDirtyLedgers()).isEmpty(); + } + + @Test + public void testMarkDirtyAtMaxValueBoundary() { + PositionRangeSet set = new PositionRangeSet(CONVERTER, true); + long near = Integer.MAX_VALUE - 1; + set.addOpenClosed(near, 0, near, 1); + Set dirty = set.snapshotAndClearDirtyLedgers(); + assertThat(dirty).containsExactly(near); + } + + @Test + public void testMarkDirtyUpperAtMaxValue() { + PositionRangeSet set = new PositionRangeSet(CONVERTER, true); + long near = Integer.MAX_VALUE - 1; + set.addOpenClosed(near - 1, 0, near, 0); + Set dirty = set.snapshotAndClearDirtyLedgers(); + assertThat(dirty).containsExactlyInAnyOrder(near - 1, near); + } +} diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetTest.java index cccb49678e682..788ed78156b7d 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionRangeSetTest.java @@ -18,6 +18,7 @@ */ package org.apache.bookkeeper.mledger.impl; +import static org.assertj.core.api.Assertions.assertThat; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNull; @@ -61,6 +62,66 @@ private static PositionRangeSet newSet() { return new PositionRangeSet(CONSUMER, true); } + @Test + public void testTotalCardinalityMaintainedIncrementally() { + PositionRangeSet set = newSet(); + assertEquals(set.totalCardinality(), 0); + + set.addOpenClosed(1, -1, 1, 4); // entries 0..4 in ledger 1 + assertEquals(set.totalCardinality(), 5); + + set.addOpenClosed(1, 4, 1, 9); // entries 5..9 in ledger 1 + assertEquals(set.totalCardinality(), 10); + + set.addOpenClosed(3, -1, 5, 2); // cross-ledger: entries 0..2 in ledger 5 + assertEquals(set.totalCardinality(), 13); + + // Overlapping add contributes nothing new. + set.addOpenClosed(1, -1, 1, 9); + assertEquals(set.totalCardinality(), 13); + + // Cross-check against the range-based cardinality computation. + assertEquals(set.cardinality(0, -1, 1_000_000, 2_000_000_000), set.totalCardinality()); + + set.removeAtMost(1, 4); // drop ledger-1 entries 0..4 + assertEquals(set.totalCardinality(), 8); + assertEquals(set.cardinality(0, -1, 1_000_000, 2_000_000_000), set.totalCardinality()); + + set.remove(Range.atMost(pos(1, 9))); // drop the rest of ledger 1 + assertEquals(set.totalCardinality(), 3); + assertEquals(set.cardinality(0, -1, 1_000_000, 2_000_000_000), set.totalCardinality()); + + set.clear(); + assertEquals(set.totalCardinality(), 0); + } + + @Test + public void testToRangesReportsTruncation() { + PositionRangeSet set = newSet(); + // ledger 1: 10 positions, ledger 2: 10 positions, ledger 3: 10 positions. + set.addOpenClosed(1, -1, 1, 9); + set.addOpenClosed(2, -1, 2, 9); + set.addOpenClosed(3, -1, 3, 9); + assertEquals(set.totalCardinality(), 30); + + // Cap that fits everything: no truncation reported. + int[] skipped = {0}; + Map all = set.toRanges(30, count -> skipped[0] = count); + assertEquals(all.size(), 3); + assertEquals(skipped[0], 0); + + // Cap that fits only two ledgers (cumulative cardinality 20): third ledger skipped + // and the notifier reports it exactly once. + skipped[0] = 0; + Map truncated = set.toRanges(20, count -> skipped[0] = count); + assertThat(truncated).containsOnlyKeys(1L, 2L); + assertEquals(skipped[0], 1); + + // Legacy overload behaves the same without a notifier. + Map legacy = set.toRanges(20); + assertThat(legacy).containsOnlyKeys(1L, 2L); + } + @Test public void testDirtyLedger() { PositionRangeSet rangeSet = newSet(); diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 371725caff3db..743ba15289166 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -2489,6 +2489,13 @@ The max allowed delay for delayed delivery (in milliseconds). If the broker rece doc = "If enabled, the maximum \"acknowledgment holes\" will not be limited and \"acknowledgment holes\" " + "are stored in multiple entries.") private boolean persistentUnackedRangesWithMultipleEntriesEnabled = false; + @FieldContext( + category = CATEGORY_STORAGE_ML, + doc = "Enables per-msgLedger cursor checkpoint persistence. When true, the cursor " + + "writes a CursorCheckpoint per flush (mark-delete ledger's ack state inline + refs " + + "to other ledgers' previously-persisted ack states) instead of the legacy single " + + "PositionInfo entry. Eliminates write amplification and avoids ack truncation.") + private boolean persistentUnackedRangesWithPerLedgerEntryEnabled = false; @Deprecated @FieldContext( category = CATEGORY_STORAGE_ML, @@ -2504,13 +2511,6 @@ The max allowed delay for delayed delivery (in milliseconds). If the broker rece + "If number of unack message range is higher than this limit then broker will persist" + " unacked ranges into bookkeeper to avoid additional data overhead into MetadataStore.") private int managedLedgerMaxUnackedRangesToPersistInMetadataStore = 1000; - @FieldContext( - dynamic = true, - category = CATEGORY_STORAGE_ML, - doc = "After enabling this feature, Pulsar will stop delivery messages to clients if the cursor metadata is" - + " too large to persist, it will help to reduce the duplicates caused by the ack state that can not be" - + " fully persistent. Default false.") - private boolean dispatcherPauseOnAckStatePersistentEnabled = false; @FieldContext( dynamic = true, category = CATEGORY_STORAGE_ML, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 5586c64ad63be..d355cb50e0a88 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -2234,6 +2234,8 @@ public CompletableFuture getManagedLedgerConfig(@NonNull To .setPersistIndividualAckAsLongArray(serviceConfig.isManagedLedgerPersistIndividualAckAsLongArray()); managedLedgerConfig.setPersistentUnackedRangesWithMultipleEntriesEnabled( serviceConfig.isPersistentUnackedRangesWithMultipleEntriesEnabled()); + managedLedgerConfig.setPersistentUnackedRangesWithPerLedgerEntryEnabled( + serviceConfig.isPersistentUnackedRangesWithPerLedgerEntryEnabled()); managedLedgerConfig.setMaxUnackedRangesToPersistInMetadataStore( serviceConfig.getManagedLedgerMaxUnackedRangesToPersistInMetadataStore()); managedLedgerConfig.setMaxEntriesPerLedger(serviceConfig.getManagedLedgerMaxEntriesPerLedger()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedCursorMetrics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedCursorMetrics.java index 639f51ead6cee..118c173cff83b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedCursorMetrics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedCursorMetrics.java @@ -77,6 +77,14 @@ private List aggregate() { metrics.put("brk_ml_cursor_writeLedgerSize", cStats.getWriteCursorLedgerSize()); metrics.put("brk_ml_cursor_writeLedgerLogicalSize", cStats.getWriteCursorLedgerLogicalSize()); metrics.put("brk_ml_cursor_readLedgerSize", cStats.getReadCursorLedgerSize()); + metrics.put("brk_ml_cursor_ackCount", cStats.getAckCount()); + metrics.put("brk_ml_cursor_ackLatencyAvgMs", (long) cStats.getAckLatencyAvgMillis()); + metrics.put("brk_ml_cursor_persistCount", cStats.getPersistCount()); + metrics.put("brk_ml_cursor_persistLatencyAvgMs", (long) cStats.getPersistLatencyAvgMillis()); + metrics.put("brk_ml_cursor_recoverCount", cStats.getRecoverCount()); + metrics.put("brk_ml_cursor_recoverSucceed", cStats.getRecoverSucceed()); + metrics.put("brk_ml_cursor_recoverErrors", cStats.getRecoverErrors()); + metrics.put("brk_ml_cursor_recoverLatencyAvgMs", (long) cStats.getRecoverLatencyAvgMillis()); metricsCollection.add(metrics); } } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ManagedLedgerInternalStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ManagedLedgerInternalStats.java index 5e5784e70681d..85829881927ce 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ManagedLedgerInternalStats.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ManagedLedgerInternalStats.java @@ -99,6 +99,9 @@ public static class CursorStats { public long messagesConsumedCounter; public long cursorLedger; public long cursorLedgerLastEntry; + public long individualDeletedMessagesCount; + public String firstIndividualDeletedMessage; + @Deprecated public String individuallyDeletedMessages; public String lastLedgerSwitchTimestamp; public String state; diff --git a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java index 7bebc0e558f7a..6813be0df319a 100644 --- a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java +++ b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java @@ -20,6 +20,7 @@ import static com.google.common.base.Preconditions.checkArgument; import com.google.common.collect.Lists; +import io.netty.util.BooleanSupplier; import io.netty.util.concurrent.DefaultThreadFactory; import java.util.ArrayList; import java.util.Arrays; @@ -201,8 +202,20 @@ public void asyncOpenLedgerNoRecovery(long lId, DigestType digestType, byte[] pa asyncOpenLedger(lId, digestType, passwd, cb, ctx); } + // Test hook: fail ledger deletions while enabled (used to exercise GC retry semantics). + private volatile java.util.function.BooleanSupplier deleteLedgerFailure; + + public void setDeleteLedgerFailure(java.util.function.BooleanSupplier deleteLedgerFailure) { + this.deleteLedgerFailure = deleteLedgerFailure; + } + @Override public void asyncDeleteLedger(long lId, DeleteCallback cb, Object ctx) { + if (deleteLedgerFailure != null && deleteLedgerFailure.getAsBoolean()) { + executor.execute(() -> + cb.deleteComplete(BKException.Code.NoBookieAvailableException, ctx)); + return; + } getProgrammedFailure().thenComposeAsync((res) -> { if (ledgers.containsKey(lId)) { ledgers.remove(lId); From 3dae4c5084708cea5d6b055e76cafe60b645a355 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Fri, 28 Aug 2026 21:42:31 +0800 Subject: [PATCH 204/213] Bump version to next snapshot version --- bouncy-castle/bc/pom.xml | 2 +- bouncy-castle/bcfips-include-test/pom.xml | 2 +- bouncy-castle/bcfips/pom.xml | 2 +- bouncy-castle/pom.xml | 2 +- buildtools/pom.xml | 2 +- distribution/io/pom.xml | 2 +- distribution/offloaders/pom.xml | 2 +- distribution/pom.xml | 2 +- distribution/server/pom.xml | 2 +- distribution/shell/pom.xml | 2 +- docker/pom.xml | 2 +- docker/pulsar-all/pom.xml | 2 +- docker/pulsar/pom.xml | 2 +- jclouds-shaded/pom.xml | 2 +- jetcd-core-shaded/pom.xml | 2 +- jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml | 2 +- jetty-upgrade/pom.xml | 2 +- jetty-upgrade/zookeeper-prometheus-metrics/pom.xml | 2 +- jetty-upgrade/zookeeper-with-patched-admin/pom.xml | 2 +- managed-ledger/pom.xml | 2 +- microbench/pom.xml | 2 +- pom.xml | 2 +- pulsar-bom/pom.xml | 2 +- pulsar-broker-auth-athenz/pom.xml | 2 +- pulsar-broker-auth-oidc/pom.xml | 2 +- pulsar-broker-auth-sasl/pom.xml | 2 +- pulsar-broker-common/pom.xml | 2 +- pulsar-broker/pom.xml | 2 +- pulsar-cli-utils/pom.xml | 2 +- pulsar-client-admin-api/pom.xml | 2 +- pulsar-client-admin-shaded/pom.xml | 2 +- pulsar-client-admin/pom.xml | 2 +- pulsar-client-all/pom.xml | 2 +- pulsar-client-api/pom.xml | 2 +- pulsar-client-auth-athenz/pom.xml | 2 +- pulsar-client-auth-sasl/pom.xml | 2 +- pulsar-client-dependencies-minimized/pom.xml | 2 +- pulsar-client-messagecrypto-bc/pom.xml | 2 +- pulsar-client-shaded/pom.xml | 2 +- pulsar-client-tools-api/pom.xml | 2 +- pulsar-client-tools-customcommand-example/pom.xml | 2 +- pulsar-client-tools-test/pom.xml | 2 +- pulsar-client-tools/pom.xml | 2 +- pulsar-client/pom.xml | 2 +- pulsar-common/pom.xml | 2 +- pulsar-config-validation/pom.xml | 2 +- pulsar-docs-tools/pom.xml | 2 +- pulsar-functions/api-java/pom.xml | 2 +- pulsar-functions/instance/pom.xml | 2 +- pulsar-functions/java-examples-builtin/pom.xml | 2 +- pulsar-functions/java-examples/pom.xml | 2 +- pulsar-functions/localrun-shaded/pom.xml | 2 +- pulsar-functions/localrun/pom.xml | 2 +- pulsar-functions/pom.xml | 2 +- pulsar-functions/proto/pom.xml | 2 +- pulsar-functions/runtime-all/pom.xml | 2 +- pulsar-functions/runtime/pom.xml | 2 +- pulsar-functions/secrets/pom.xml | 2 +- pulsar-functions/utils/pom.xml | 2 +- pulsar-functions/worker/pom.xml | 2 +- pulsar-io/aerospike/pom.xml | 2 +- pulsar-io/alluxio/pom.xml | 2 +- pulsar-io/aws/pom.xml | 2 +- pulsar-io/azure-data-explorer/pom.xml | 2 +- pulsar-io/batch-data-generator/pom.xml | 2 +- pulsar-io/batch-discovery-triggerers/pom.xml | 2 +- pulsar-io/canal/pom.xml | 2 +- pulsar-io/cassandra/pom.xml | 2 +- pulsar-io/common/pom.xml | 2 +- pulsar-io/core/pom.xml | 2 +- pulsar-io/data-generator/pom.xml | 2 +- pulsar-io/debezium/core/pom.xml | 2 +- pulsar-io/debezium/mongodb/pom.xml | 2 +- pulsar-io/debezium/mssql/pom.xml | 2 +- pulsar-io/debezium/mysql/pom.xml | 2 +- pulsar-io/debezium/oracle/pom.xml | 2 +- pulsar-io/debezium/pom.xml | 2 +- pulsar-io/debezium/postgres/pom.xml | 2 +- pulsar-io/docs/pom.xml | 2 +- pulsar-io/dynamodb/pom.xml | 2 +- pulsar-io/elastic-search/pom.xml | 2 +- pulsar-io/file/pom.xml | 2 +- pulsar-io/flume/pom.xml | 2 +- pulsar-io/hbase/pom.xml | 2 +- pulsar-io/hdfs3/pom.xml | 2 +- pulsar-io/http/pom.xml | 2 +- pulsar-io/influxdb/pom.xml | 2 +- pulsar-io/jdbc/clickhouse/pom.xml | 2 +- pulsar-io/jdbc/core/pom.xml | 2 +- pulsar-io/jdbc/mariadb/pom.xml | 2 +- pulsar-io/jdbc/openmldb/pom.xml | 2 +- pulsar-io/jdbc/pom.xml | 2 +- pulsar-io/jdbc/postgres/pom.xml | 2 +- pulsar-io/jdbc/sqlite/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor-nar/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor/pom.xml | 2 +- pulsar-io/kafka/pom.xml | 2 +- pulsar-io/kinesis-kpl-shaded/pom.xml | 2 +- pulsar-io/kinesis/pom.xml | 2 +- pulsar-io/mongo/pom.xml | 2 +- pulsar-io/netty/pom.xml | 2 +- pulsar-io/nsq/pom.xml | 2 +- pulsar-io/pom.xml | 2 +- pulsar-io/rabbitmq/pom.xml | 2 +- pulsar-io/redis/pom.xml | 2 +- pulsar-io/solr/pom.xml | 2 +- pulsar-io/twitter/pom.xml | 2 +- pulsar-metadata/pom.xml | 2 +- pulsar-opentelemetry/pom.xml | 2 +- pulsar-package-management/bookkeeper-storage/pom.xml | 2 +- pulsar-package-management/core/pom.xml | 2 +- pulsar-package-management/filesystem-storage/pom.xml | 2 +- pulsar-package-management/pom.xml | 2 +- pulsar-proxy/pom.xml | 2 +- pulsar-testclient/pom.xml | 2 +- pulsar-transaction/common/pom.xml | 2 +- pulsar-transaction/coordinator/pom.xml | 2 +- pulsar-transaction/pom.xml | 2 +- pulsar-websocket/pom.xml | 2 +- structured-event-log/pom.xml | 2 +- testmocks/pom.xml | 2 +- tests/bc_2_0_0/pom.xml | 2 +- tests/bc_2_0_1/pom.xml | 2 +- tests/bc_2_6_0/pom.xml | 2 +- tests/docker-images/java-test-functions/pom.xml | 2 +- tests/docker-images/java-test-image/pom.xml | 2 +- tests/docker-images/java-test-plugins/pom.xml | 2 +- tests/docker-images/latest-version-image/pom.xml | 2 +- tests/docker-images/pom.xml | 2 +- tests/integration/pom.xml | 2 +- tests/pom.xml | 2 +- tests/pulsar-client-admin-shade-test/pom.xml | 2 +- tests/pulsar-client-all-shade-test/pom.xml | 2 +- tests/pulsar-client-shade-test/pom.xml | 2 +- tiered-storage/file-system/pom.xml | 2 +- tiered-storage/jcloud/pom.xml | 2 +- tiered-storage/pom.xml | 2 +- 137 files changed, 137 insertions(+), 137 deletions(-) diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index cdbbbe5ddad9a..d75896224ca5e 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -25,7 +25,7 @@ com.ascentstream.pulsar bouncy-castle-parent - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT bouncy-castle-bc diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index 0d1229f92074e..82f3336bdced7 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -24,7 +24,7 @@ com.ascentstream.pulsar bouncy-castle-parent - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT bcfips-include-test diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index 8900500cb92ba..c0b79f96e68f3 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -25,7 +25,7 @@ com.ascentstream.pulsar bouncy-castle-parent - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT bouncy-castle-bcfips diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index 3fa28ee5f8006..f99317dcb9fd3 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -25,7 +25,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT diff --git a/buildtools/pom.xml b/buildtools/pom.xml index f083c563a6288..27bf5434da9c7 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -31,7 +31,7 @@ com.ascentstream.pulsar buildtools - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT jar Pulsar Build Tools diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index 107e36ee622ef..e4ac7cda2e19b 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -25,7 +25,7 @@ com.ascentstream.pulsar distribution - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-io-distribution diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index 980cbfe7b4bd1..66a7316867f91 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -25,7 +25,7 @@ com.ascentstream.pulsar distribution - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-offloader-distribution diff --git a/distribution/pom.xml b/distribution/pom.xml index f44e2db01f654..dae2fb0469c94 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -25,7 +25,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT distribution diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 02911df137cd5..0d88017b1bd59 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -25,7 +25,7 @@ com.ascentstream.pulsar distribution - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-server-distribution diff --git a/distribution/shell/pom.xml b/distribution/shell/pom.xml index c85e18e85e597..a4efb54528ba4 100644 --- a/distribution/shell/pom.xml +++ b/distribution/shell/pom.xml @@ -25,7 +25,7 @@ com.ascentstream.pulsar distribution - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-shell-distribution diff --git a/docker/pom.xml b/docker/pom.xml index 5d49f46da16ca..4664b212858bc 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -26,7 +26,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT docker-images Apache Pulsar :: Docker Images diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index 415f3587b1c7d..c6b9e46eb7ed1 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -23,7 +23,7 @@ com.ascentstream.pulsar docker-images - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT 4.0.0 pulsar-all-docker-image diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index b8de1321d1384..0e60acc65299d 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -23,7 +23,7 @@ com.ascentstream.pulsar docker-images - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT 4.0.0 pulsar-docker-image diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index 21d58c059d4ba..dcebbc7a5f7ad 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -26,7 +26,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT jclouds-shaded diff --git a/jetcd-core-shaded/pom.xml b/jetcd-core-shaded/pom.xml index faff391d4edde..c45644f5d7c25 100644 --- a/jetcd-core-shaded/pom.xml +++ b/jetcd-core-shaded/pom.xml @@ -26,7 +26,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT jetcd-core-shaded diff --git a/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml b/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml index b224c7545e2d1..6336d6c27f56a 100644 --- a/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml +++ b/jetty-upgrade/bookkeeper-prometheus-metrics-provider/pom.xml @@ -24,7 +24,7 @@ com.ascentstream.pulsar jetty-upgrade - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-bookkeeper-prometheus-metrics-provider Apache Pulsar :: BookKeeper Stats Providers :: Prometheus diff --git a/jetty-upgrade/pom.xml b/jetty-upgrade/pom.xml index 7e05cfa623486..c5e190ff76cf9 100644 --- a/jetty-upgrade/pom.xml +++ b/jetty-upgrade/pom.xml @@ -25,7 +25,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT jetty-upgrade diff --git a/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml b/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml index 8805d5fd5bef8..15c6a99722448 100755 --- a/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml +++ b/jetty-upgrade/zookeeper-prometheus-metrics/pom.xml @@ -24,7 +24,7 @@ com.ascentstream.pulsar jetty-upgrade - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-zookeeper-prometheus-metrics diff --git a/jetty-upgrade/zookeeper-with-patched-admin/pom.xml b/jetty-upgrade/zookeeper-with-patched-admin/pom.xml index 303675f4db533..81c0f1867674d 100644 --- a/jetty-upgrade/zookeeper-with-patched-admin/pom.xml +++ b/jetty-upgrade/zookeeper-with-patched-admin/pom.xml @@ -26,7 +26,7 @@ com.ascentstream.pulsar jetty-upgrade - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT zookeeper-with-patched-admin diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index e2f16c5561242..7fb47c6d9e220 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -25,7 +25,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT managed-ledger diff --git a/microbench/pom.xml b/microbench/pom.xml index d0b8fd129102a..8eb8c08131a41 100644 --- a/microbench/pom.xml +++ b/microbench/pom.xml @@ -25,7 +25,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT microbench diff --git a/pom.xml b/pom.xml index 46a6607dab037..42a405e6c6e7e 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT Pulsar Pulsar is a distributed pub-sub messaging platform with a very diff --git a/pulsar-bom/pom.xml b/pulsar-bom/pom.xml index 8236b2f87e035..dc6a87264387f 100644 --- a/pulsar-bom/pom.xml +++ b/pulsar-bom/pom.xml @@ -33,7 +33,7 @@ com.ascentstream.pulsar pulsar-bom - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT Pulsar BOM Pulsar (Bill of Materials) diff --git a/pulsar-broker-auth-athenz/pom.xml b/pulsar-broker-auth-athenz/pom.xml index 401543e8f129e..5740b9912da5b 100644 --- a/pulsar-broker-auth-athenz/pom.xml +++ b/pulsar-broker-auth-athenz/pom.xml @@ -26,7 +26,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-broker-auth-athenz diff --git a/pulsar-broker-auth-oidc/pom.xml b/pulsar-broker-auth-oidc/pom.xml index 9beb4143c7acb..4136d46cf88cb 100644 --- a/pulsar-broker-auth-oidc/pom.xml +++ b/pulsar-broker-auth-oidc/pom.xml @@ -26,7 +26,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-broker-auth-oidc diff --git a/pulsar-broker-auth-sasl/pom.xml b/pulsar-broker-auth-sasl/pom.xml index a09cae8bb6d45..49a95e66e91ca 100644 --- a/pulsar-broker-auth-sasl/pom.xml +++ b/pulsar-broker-auth-sasl/pom.xml @@ -26,7 +26,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-broker-auth-sasl diff --git a/pulsar-broker-common/pom.xml b/pulsar-broker-common/pom.xml index 30c3714bef556..b0a27b1269089 100644 --- a/pulsar-broker-common/pom.xml +++ b/pulsar-broker-common/pom.xml @@ -26,7 +26,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-broker-common diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 201114f5e2028..a68768a86c9ca 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -25,7 +25,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-broker diff --git a/pulsar-cli-utils/pom.xml b/pulsar-cli-utils/pom.xml index ae319bdfb125a..d7c19798bb596 100644 --- a/pulsar-cli-utils/pom.xml +++ b/pulsar-cli-utils/pom.xml @@ -26,7 +26,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-cli-utils diff --git a/pulsar-client-admin-api/pom.xml b/pulsar-client-admin-api/pom.xml index 1196a932007b7..92d3e19b2e5f6 100644 --- a/pulsar-client-admin-api/pom.xml +++ b/pulsar-client-admin-api/pom.xml @@ -25,7 +25,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-client-admin-api diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml index 13ab2777bdfd8..65e40395f30f1 100644 --- a/pulsar-client-admin-shaded/pom.xml +++ b/pulsar-client-admin-shaded/pom.xml @@ -25,7 +25,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-client-admin Pulsar Client Admin diff --git a/pulsar-client-admin/pom.xml b/pulsar-client-admin/pom.xml index ff75c7e87a716..59c0328f996bc 100644 --- a/pulsar-client-admin/pom.xml +++ b/pulsar-client-admin/pom.xml @@ -26,7 +26,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-client-admin-original diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index 0b7bb0f781090..40ffb7bf11212 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -25,7 +25,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-client-all Pulsar Client All diff --git a/pulsar-client-api/pom.xml b/pulsar-client-api/pom.xml index 7ccbbf49e6d9d..398fe265b7689 100644 --- a/pulsar-client-api/pom.xml +++ b/pulsar-client-api/pom.xml @@ -25,7 +25,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-client-api diff --git a/pulsar-client-auth-athenz/pom.xml b/pulsar-client-auth-athenz/pom.xml index 907cb08502157..96ac32158bd33 100644 --- a/pulsar-client-auth-athenz/pom.xml +++ b/pulsar-client-auth-athenz/pom.xml @@ -25,7 +25,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-client-auth-athenz diff --git a/pulsar-client-auth-sasl/pom.xml b/pulsar-client-auth-sasl/pom.xml index 496e469c4c020..5f5c358878794 100644 --- a/pulsar-client-auth-sasl/pom.xml +++ b/pulsar-client-auth-sasl/pom.xml @@ -25,7 +25,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-client-auth-sasl diff --git a/pulsar-client-dependencies-minimized/pom.xml b/pulsar-client-dependencies-minimized/pom.xml index 0ac7112fac66f..13ed1e3787ed7 100644 --- a/pulsar-client-dependencies-minimized/pom.xml +++ b/pulsar-client-dependencies-minimized/pom.xml @@ -26,7 +26,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-client-dependencies-minimized diff --git a/pulsar-client-messagecrypto-bc/pom.xml b/pulsar-client-messagecrypto-bc/pom.xml index c19c41648e0d2..4e3798430b463 100644 --- a/pulsar-client-messagecrypto-bc/pom.xml +++ b/pulsar-client-messagecrypto-bc/pom.xml @@ -25,7 +25,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-client-messagecrypto-bc diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml index 02ad0d09a7eab..0e69532c6a818 100644 --- a/pulsar-client-shaded/pom.xml +++ b/pulsar-client-shaded/pom.xml @@ -25,7 +25,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-client Pulsar Client Java diff --git a/pulsar-client-tools-api/pom.xml b/pulsar-client-tools-api/pom.xml index 358085be2f5f7..5b2155544a905 100644 --- a/pulsar-client-tools-api/pom.xml +++ b/pulsar-client-tools-api/pom.xml @@ -24,7 +24,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-client-tools-api diff --git a/pulsar-client-tools-customcommand-example/pom.xml b/pulsar-client-tools-customcommand-example/pom.xml index 3c5482e7e8abe..a1f8d5bb17e12 100644 --- a/pulsar-client-tools-customcommand-example/pom.xml +++ b/pulsar-client-tools-customcommand-example/pom.xml @@ -22,7 +22,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT 4.0.0 pulsar-client-tools-customcommand-example diff --git a/pulsar-client-tools-test/pom.xml b/pulsar-client-tools-test/pom.xml index a9ac5b7f215b5..a1729152761ca 100644 --- a/pulsar-client-tools-test/pom.xml +++ b/pulsar-client-tools-test/pom.xml @@ -24,7 +24,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-client-tools-test diff --git a/pulsar-client-tools/pom.xml b/pulsar-client-tools/pom.xml index f1028ae2f4c3e..797d975f8c79c 100644 --- a/pulsar-client-tools/pom.xml +++ b/pulsar-client-tools/pom.xml @@ -24,7 +24,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-client-tools diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index 56d5638b2100a..901bb3701c727 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -25,7 +25,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-client-original diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index 638ed8506f3c3..b01ce5ae2dfd6 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -26,7 +26,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-common diff --git a/pulsar-config-validation/pom.xml b/pulsar-config-validation/pom.xml index 0c211b3d24845..44656f646186a 100644 --- a/pulsar-config-validation/pom.xml +++ b/pulsar-config-validation/pom.xml @@ -26,7 +26,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-config-validation diff --git a/pulsar-docs-tools/pom.xml b/pulsar-docs-tools/pom.xml index da03777628dcd..ddd311417698c 100644 --- a/pulsar-docs-tools/pom.xml +++ b/pulsar-docs-tools/pom.xml @@ -27,7 +27,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-docs-tools diff --git a/pulsar-functions/api-java/pom.xml b/pulsar-functions/api-java/pom.xml index 1ce7bd27c00f5..0063b1dbdbfa6 100644 --- a/pulsar-functions/api-java/pom.xml +++ b/pulsar-functions/api-java/pom.xml @@ -24,7 +24,7 @@ com.ascentstream.pulsar pulsar-functions - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-functions-api diff --git a/pulsar-functions/instance/pom.xml b/pulsar-functions/instance/pom.xml index 60a7a4ade072a..603bf4c198c8c 100644 --- a/pulsar-functions/instance/pom.xml +++ b/pulsar-functions/instance/pom.xml @@ -25,7 +25,7 @@ com.ascentstream.pulsar pulsar-functions - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-functions-instance diff --git a/pulsar-functions/java-examples-builtin/pom.xml b/pulsar-functions/java-examples-builtin/pom.xml index 7804e2ead6014..93035c454a9ab 100644 --- a/pulsar-functions/java-examples-builtin/pom.xml +++ b/pulsar-functions/java-examples-builtin/pom.xml @@ -24,7 +24,7 @@ com.ascentstream.pulsar pulsar-functions - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-functions-api-examples-builtin diff --git a/pulsar-functions/java-examples/pom.xml b/pulsar-functions/java-examples/pom.xml index 01468c7b9c7e8..37b7269aaa9ff 100644 --- a/pulsar-functions/java-examples/pom.xml +++ b/pulsar-functions/java-examples/pom.xml @@ -24,7 +24,7 @@ com.ascentstream.pulsar pulsar-functions - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-functions-api-examples diff --git a/pulsar-functions/localrun-shaded/pom.xml b/pulsar-functions/localrun-shaded/pom.xml index b0175f04fb52d..c94d7e35e4c35 100644 --- a/pulsar-functions/localrun-shaded/pom.xml +++ b/pulsar-functions/localrun-shaded/pom.xml @@ -26,7 +26,7 @@ com.ascentstream.pulsar pulsar-functions - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-functions-local-runner diff --git a/pulsar-functions/localrun/pom.xml b/pulsar-functions/localrun/pom.xml index de026f08fc665..edf2a4bf52de6 100644 --- a/pulsar-functions/localrun/pom.xml +++ b/pulsar-functions/localrun/pom.xml @@ -26,7 +26,7 @@ com.ascentstream.pulsar pulsar-functions - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-functions-local-runner-original diff --git a/pulsar-functions/pom.xml b/pulsar-functions/pom.xml index 4ea6ea2c708a0..c6b20cf55ee10 100644 --- a/pulsar-functions/pom.xml +++ b/pulsar-functions/pom.xml @@ -25,7 +25,7 @@ com.ascentstream.pulsar pulsar - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-functions diff --git a/pulsar-functions/proto/pom.xml b/pulsar-functions/proto/pom.xml index 2c9a7e08e951d..62b07b808008c 100644 --- a/pulsar-functions/proto/pom.xml +++ b/pulsar-functions/proto/pom.xml @@ -27,7 +27,7 @@ com.ascentstream.pulsar pulsar-functions - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT pulsar-functions-proto diff --git a/pulsar-functions/runtime-all/pom.xml b/pulsar-functions/runtime-all/pom.xml index 880156a7f8fb4..695398e332abf 100644 --- a/pulsar-functions/runtime-all/pom.xml +++ b/pulsar-functions/runtime-all/pom.xml @@ -26,7 +26,7 @@ com.ascentstream.pulsar pulsar-functions - 4.0.11.1-226281f-20260709031812-SNAPSHOT + 4.0.14-SNAPSHOT