io.dropwizard.metrics
metrics-core
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java
index 4e5e12365480c..659b162a7658c 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java
@@ -18,12 +18,14 @@
*/
package org.apache.bookkeeper.mledger;
+import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables;
import com.google.common.collect.Range;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
import org.apache.bookkeeper.common.annotation.InterfaceAudience;
import org.apache.bookkeeper.common.annotation.InterfaceStability;
@@ -369,6 +371,25 @@ default void asyncReadEntriesWithSkipOrWait(int maxEntries, long maxSizeBytes, R
*/
long getNumberOfEntriesInBacklog(boolean isPrecise);
+ /**
+ * Return whether this cursor has non-deleted messages in backlog.
+ *
+ * @return true if there is at least one entry in backlog
+ */
+ default boolean hasBacklog() {
+ return hasBacklog(true);
+ }
+
+ /**
+ * Return whether this cursor has non-deleted messages in backlog.
+ *
+ * @param isPrecise set to true to get a precise backlog check
+ * @return true if there is at least one entry in backlog
+ */
+ default boolean hasBacklog(boolean isPrecise) {
+ return getNumberOfEntriesInBacklog(isPrecise) > 0;
+ }
+
/**
* This signals that the reader is done with all the entries up to "position" (included). This can potentially
* trigger a ledger deletion, if all the other cursors are done too with the underlying ledger.
@@ -860,6 +881,20 @@ default void skipNonRecoverableLedger(long ledgerId){}
*/
ManagedLedger getManagedLedger();
+ /**
+ * Schedule a continuation of a read callback.
+ *
+ * Implementations that deliver read callbacks on a dedicated execution context should override this method
+ * to run the continuation on that same execution context.
+ *
+ * @param callback the callback continuation
+ * @param delay the delay before executing the continuation
+ * @param unit the time unit of the delay
+ */
+ default void scheduleReadCallback(Runnable callback, long delay, TimeUnit unit) {
+ CompletableFuture.delayedExecutor(delay, unit).execute(catchingAndLoggingThrowables(callback));
+ }
+
/**
* Get last individual deleted range.
* @return range
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursorMXBean.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursorMXBean.java
index 7402bd65f793e..dc04fc4923429 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursorMXBean.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursorMXBean.java
@@ -100,4 +100,66 @@ public interface ManagedCursorMXBean {
*/
long getReadCursorLedgerSize();
+ /**
+ * Record an acknowledgment operation and its processing latency.
+ *
+ * @param latencyMillis acknowledgment processing latency in milliseconds
+ */
+ void recordAck(long latencyMillis);
+
+ /**
+ * @return the number of acknowledgment operations
+ */
+ long getAckCount();
+
+ /**
+ * @return the average acknowledgment processing latency in milliseconds
+ */
+ double getAckLatencyAvgMillis();
+
+ /**
+ * Record a cursor persist (checkpoint write to the cursor ledger) operation and its latency.
+ *
+ * @param latencyMillis persist latency in milliseconds
+ */
+ void recordPersist(long latencyMillis);
+
+ /**
+ * @return the number of cursor persist operations
+ */
+ long getPersistCount();
+
+ /**
+ * @return the average cursor persist latency in milliseconds
+ */
+ double getPersistLatencyAvgMillis();
+
+ /**
+ * Record a cursor recovery operation and its latency.
+ *
+ * @param latencyMillis recovery latency in milliseconds
+ * @param success whether the recovery completed successfully
+ */
+ void recordRecover(long latencyMillis, boolean success);
+
+ /**
+ * @return the number of cursor recovery operations
+ */
+ long getRecoverCount();
+
+ /**
+ * @return the number of successful cursor recovery operations
+ */
+ long getRecoverSucceed();
+
+ /**
+ * @return the number of failed cursor recovery operations
+ */
+ long getRecoverErrors();
+
+ /**
+ * @return the average cursor recovery latency in milliseconds
+ */
+ double getRecoverLatencyAvgMillis();
+
}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java
index aaa127973a5ec..eb3d4844d6d48 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java
@@ -33,7 +33,6 @@
import org.apache.bookkeeper.mledger.impl.NullLedgerOffloader;
import org.apache.bookkeeper.mledger.intercept.ManagedLedgerInterceptor;
import org.apache.commons.collections4.MapUtils;
-import org.apache.pulsar.common.util.collections.OpenLongPairRangeSet;
/**
* Configuration class for a ManagedLedger.
@@ -46,6 +45,22 @@ public class ManagedLedgerConfig {
private int maxUnackedRangesToPersist = 10000;
private int maxBatchDeletedIndexToPersist = 10000;
private boolean persistentUnackedRangesWithMultipleEntriesEnabled = false;
+ /**
+ * Enables per-msgLedger cursor checkpoint persistence. When true, the cursor
+ * writes one {@code CursorCheckpoint} per flush (with the mark-delete ledger's ack state
+ * inline and refs to other ledgers' previously-persisted ack states) instead of the
+ * legacy single-{@code PositionInfo} entry.
+ *
+ *
See {@code CursorLogEntry}, {@code CursorCheckpoint}, {@code AckState}, and
+ * {@code AckStateRef} in {@code MLDataFormats.proto} for the wire format.
+ */
+ private boolean persistentUnackedRangesWithPerLedgerEntryEnabled = false;
+ /**
+ * Maximum serialized size for a single {@code CursorLogEntry}. Entries exceeding
+ * this are transparently chunked. Not exposed as a broker config — defaults to the same
+ * 5 MB as {@code maxMessageSize} so chunk behavior mirrors the message-size limit.
+ */
+ private int persistentUnackedRangesMaxEntrySize = 5 * 1024 * 1024;
private boolean deletionAtBatchIndexLevelEnabled = true;
private int maxUnackedRangesToPersistInMetadataStore = 1000;
private int maxEntriesPerLedger = 50000;
@@ -71,7 +86,6 @@ public class ManagedLedgerConfig {
private long addEntryTimeoutSeconds = 120;
private DigestType digestType = DigestType.CRC32C;
private byte[] password = "".getBytes(StandardCharsets.UTF_8);
- private boolean unackedRangesOpenCacheSetEnabled = true;
private Class bookKeeperEnsemblePlacementPolicyClassName;
private Map bookKeeperEnsemblePlacementPolicyProperties;
private LedgerOffloader ledgerOffloader = NullLedgerOffloader.INSTANCE;
@@ -292,19 +306,6 @@ public ManagedLedgerConfig setPassword(String password) {
return this;
}
- /**
- * should use {@link OpenLongPairRangeSet} to store unacked ranges.
- * @return
- */
- public boolean isUnackedRangesOpenCacheSetEnabled() {
- return unackedRangesOpenCacheSetEnabled;
- }
-
- public ManagedLedgerConfig setUnackedRangesOpenCacheSetEnabled(boolean unackedRangesOpenCacheSetEnabled) {
- this.unackedRangesOpenCacheSetEnabled = unackedRangesOpenCacheSetEnabled;
- return this;
- }
-
/**
* @return the metadataEnsemblesize
*/
@@ -520,6 +521,22 @@ public void setPersistentUnackedRangesWithMultipleEntriesEnabled(boolean multipl
this.persistentUnackedRangesWithMultipleEntriesEnabled = multipleEntriesEnabled;
}
+ public boolean isPersistentUnackedRangesWithPerLedgerEntryEnabled() {
+ return persistentUnackedRangesWithPerLedgerEntryEnabled;
+ }
+
+ public void setPersistentUnackedRangesWithPerLedgerEntryEnabled(boolean enabled) {
+ this.persistentUnackedRangesWithPerLedgerEntryEnabled = enabled;
+ }
+
+ public int getPersistentUnackedRangesMaxEntrySize() {
+ return persistentUnackedRangesMaxEntrySize;
+ }
+
+ public void setPersistentUnackedRangesMaxEntrySize(int maxEntrySize) {
+ this.persistentUnackedRangesMaxEntrySize = maxEntrySize;
+ }
+
/**
* @param maxUnackedRangesToPersist
* max unacked message ranges that will be persisted and receverd.
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AutomaticOffloadTriggerController.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AutomaticOffloadTriggerController.java
new file mode 100644
index 0000000000000..35f5a26706371
--- /dev/null
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AutomaticOffloadTriggerController.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger.impl;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Coalesces repeated automatic offload triggers into at most one active run and one follow-up run.
+ */
+final class AutomaticOffloadTriggerController {
+ private static final int IDLE = 0;
+ private static final int RUNNING = 1;
+ private static final int RUNNING_WITH_PENDING_TRIGGER = 2;
+
+ private final AtomicInteger state = new AtomicInteger(IDLE);
+
+ /**
+ * Records an automatic offload trigger.
+ *
+ * @return true when the caller must start a new automatic offload run
+ */
+ boolean requestRun() {
+ while (true) {
+ int current = state.get();
+ switch (current) {
+ case IDLE:
+ if (state.compareAndSet(IDLE, RUNNING)) {
+ return true;
+ }
+ break;
+ case RUNNING:
+ if (state.compareAndSet(RUNNING, RUNNING_WITH_PENDING_TRIGGER)) {
+ return false;
+ }
+ break;
+ case RUNNING_WITH_PENDING_TRIGGER:
+ return false;
+ default:
+ throw new IllegalStateException("Unknown automatic offload trigger state: " + current);
+ }
+ }
+ }
+
+ /**
+ * Records completion of the current automatic offload run.
+ *
+ * @return true when the caller must immediately start one coalesced follow-up run
+ */
+ boolean completeRun() {
+ while (true) {
+ int current = state.get();
+ switch (current) {
+ case IDLE:
+ return false;
+ case RUNNING:
+ if (state.compareAndSet(RUNNING, IDLE)) {
+ return false;
+ }
+ break;
+ case RUNNING_WITH_PENDING_TRIGGER:
+ if (state.compareAndSet(RUNNING_WITH_PENDING_TRIGGER, RUNNING)) {
+ return true;
+ }
+ break;
+ default:
+ throw new IllegalStateException("Unknown automatic offload trigger state: " + current);
+ }
+ }
+ }
+}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointLog.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointLog.java
new file mode 100644
index 0000000000000..fdaedfbcf150f
--- /dev/null
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointLog.java
@@ -0,0 +1,416 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger.impl;
+
+import com.google.protobuf.ByteString;
+import java.io.ByteArrayOutputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.client.BKException;
+import org.apache.bookkeeper.client.LedgerHandle;
+import org.apache.bookkeeper.mledger.ManagedLedgerException;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats.CursorCheckpoint;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats.CursorCheckpointChunk;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats.CursorLogEntry;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats.PositionInfo;
+import org.apache.pulsar.common.util.FutureUtil;
+
+@Slf4j
+class CursorCheckpointLog {
+
+ private static final long MAX_SCAN_BACK = 1000L;
+
+ private final int maxEntrySize;
+ private final int chunkEnvelopeOverhead;
+ private volatile long zkCheckpointLedgerId = -1;
+ private volatile long zkCheckpointEntryId = -1;
+
+ CursorCheckpointLog(int maxEntrySize) {
+ if (maxEntrySize < 1024) {
+ throw new IllegalArgumentException("maxEntrySize must be at least 1024 bytes");
+ }
+ this.maxEntrySize = maxEntrySize;
+ CursorLogEntry probe = CursorLogEntry.newBuilder()
+ .setCheckpointChunk(CursorCheckpointChunk.newBuilder()
+ .setPartIndex(0).setPartCount(1)
+ .setCheckpointBytes(ByteString.copyFrom(new byte[0]))
+ .build())
+ .build();
+ this.chunkEnvelopeOverhead = probe.toByteArray().length + 8;
+ }
+
+ void setZkCheckpointHint(long cursorLedgerId, long entryId) {
+ this.zkCheckpointLedgerId = cursorLedgerId;
+ this.zkCheckpointEntryId = entryId;
+ }
+
+ CompletableFuture appendCheckpoint(LedgerHandle lh, CursorCheckpoint checkpoint) {
+ byte[] checkpointBytes = checkpoint.toByteArray();
+ CursorLogEntry envelope;
+ try {
+ envelope = CursorLogEntry.newBuilder()
+ .setCheckpoint(CursorCheckpoint.parseFrom(checkpointBytes))
+ .build();
+ } catch (Exception e) {
+ return FutureUtil.failedFuture(new ManagedLedgerException("Failed to parse checkpoint", e));
+ }
+ byte[] data = envelope.toByteArray();
+ if (data.length <= maxEntrySize) {
+ return addEntry(lh, data).thenApply(entryId -> {
+ log.debug("Appended checkpoint, ledgerId: {}, entryId: {}, size: {}",
+ lh.getId(), entryId, data.length);
+ return new AppendResult(data.length, entryId);
+ });
+ }
+ int maxPayloadSize = maxEntrySize - chunkEnvelopeOverhead;
+ int partCount = (checkpointBytes.length + maxPayloadSize - 1) / maxPayloadSize;
+ log.debug("Appending chunked checkpoint, ledgerId: {}, checkpointBytes: {}, partCount: {}",
+ lh.getId(), checkpointBytes.length, partCount);
+ List> futures = new ArrayList<>(partCount);
+ int offset = 0;
+ for (int i = 0; i < partCount; i++) {
+ int length = Math.min(maxPayloadSize, checkpointBytes.length - offset);
+ byte[] payload = new byte[length];
+ System.arraycopy(checkpointBytes, offset, payload, 0, length);
+ offset += length;
+ CursorLogEntry part = CursorLogEntry.newBuilder()
+ .setCheckpointChunk(CursorCheckpointChunk.newBuilder()
+ .setPartIndex(i).setPartCount(partCount)
+ .setCheckpointBytes(ByteString.copyFrom(payload))
+ .build())
+ .build();
+ futures.add(addEntry(lh, part.toByteArray()));
+ }
+ CompletableFuture lastPartFuture = futures.get(partCount - 1);
+ return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]))
+ .thenCompose(ignored -> lastPartFuture
+ .thenApply(lastEntryId -> {
+ log.debug("Appended chunked checkpoint, ledgerId: {}, lastEntryId: {}, "
+ + "partCount: {}, totalBytes: {}",
+ lh.getId(), lastEntryId, partCount, checkpointBytes.length);
+ return new AppendResult(checkpointBytes.length, lastEntryId);
+ }));
+ }
+
+ CompletableFuture readLatest(LedgerHandle lh) {
+ long last = lh.getLastAddConfirmed();
+ if (last < 0) {
+ return FutureUtil.failedFuture(new ManagedLedgerException(
+ "Cursor ledger " + lh.getId() + " has no entries"));
+ }
+ log.debug("Recovering checkpoint, ledgerId: {}, lastEntryId: {}", lh.getId(), last);
+ return recoverEntryAt(lh, last, true).thenCompose(decision -> {
+ if (decision.shouldScanBack) {
+ log.info("Scanning back for last complete checkpoint, ledgerId: {}, entryId: {}",
+ lh.getId(), last);
+ return scanBack(lh, last);
+ }
+ return CompletableFuture.completedFuture(decision.state);
+ });
+ }
+
+ /**
+ * Recovers the checkpoint stored at a specific entry, assembling chunked checkpoints when
+ * needed. Unlike {@link #readLatest}, this never scans back: the target of an ack state ref
+ * must be a complete checkpoint at exactly the given entry, so anything else fails fast.
+ */
+ CompletableFuture readAt(LedgerHandle lh, long entryId) {
+ return recoverEntryAt(lh, entryId, false).thenCompose(decision -> {
+ if (decision.shouldScanBack) {
+ return FutureUtil.failedFuture(new ManagedLedgerException(
+ "No complete checkpoint at entry " + entryId + " in ledger " + lh.getId()));
+ }
+ return CompletableFuture.completedFuture(decision.state);
+ });
+ }
+
+ /**
+ * Reads and decodes the entry at {@code entryId}. When {@code fallbackToScanBack} is set,
+ * decode failures yield a scan-back decision (used by the tail/scan-back paths); otherwise they
+ * propagate to the caller.
+ */
+ private CompletableFuture recoverEntryAt(LedgerHandle lh, long entryId,
+ boolean fallbackToScanBack) {
+ CompletableFuture recovered = readEntry(lh, entryId)
+ .thenCompose(bytes -> recoverEntry(lh, entryId, bytes));
+ return fallbackToScanBack
+ ? recovered.exceptionally(error -> {
+ log.warn("Failed to recover entry, ledgerId: {}, entryId: {}", lh.getId(), entryId, error);
+ return RecoveryDecision.scanBack();
+ })
+ : recovered;
+ }
+
+ private CompletableFuture recoverEntry(LedgerHandle lh, long entryId, byte[] bytes) {
+ if (bytes == null) {
+ return FutureUtil.failedFuture(new ManagedLedgerException(
+ "Entry " + entryId + " in cursor ledger " + lh.getId() + " is empty"));
+ }
+
+ CursorLogEntry envelope;
+ try {
+ envelope = CursorLogEntry.parseFrom(bytes);
+ } catch (Exception e) {
+ if (isLegacyPositionInfo(bytes)) {
+ // Legacy PositionInfo bytes can fail CursorLogEntry parsing.
+ log.debug("Recovered legacy PositionInfo after CursorLogEntry parse failure, "
+ + "ledgerId: {}, entryId: {}", lh.getId(), entryId);
+ return CompletableFuture.completedFuture(RecoveryDecision.recovered(RecoveredState.legacy(bytes)));
+ }
+ return FutureUtil.failedFuture(new ManagedLedgerException(
+ "Invalid cursor log entry at ledger " + lh.getId() + " entry " + entryId
+ + ": failed to parse CursorLogEntry", e));
+ }
+ if (envelope.hasCheckpoint()) {
+ CursorCheckpoint cp = envelope.getCheckpoint();
+ log.debug("Recovered checkpoint, ledgerId: {}, entryId: {}, mdLedgerId: {}, mdEntryId: {}, "
+ + "ackStates: {}, ackStateRefs: {}",
+ lh.getId(), entryId, cp.getMarkDeleteLedgerId(), cp.getMarkDeleteEntryId(),
+ cp.getAckStatesCount(), cp.getAckStateRefsCount());
+ return CompletableFuture.completedFuture(
+ RecoveryDecision.recovered(new RecoveredState(cp, entryId)));
+ }
+ if (envelope.hasCheckpointChunk()) {
+ CursorCheckpointChunk chunk = envelope.getCheckpointChunk();
+ int partCount = chunk.getPartCount();
+ int partIndex = chunk.getPartIndex();
+ if (partIndex >= 0 && partIndex < partCount) {
+ return recoverFromChunk(lh, entryId, chunk);
+ }
+ }
+ if (isLegacyPositionInfo(bytes)) {
+ // Backward compatibility: legacy PositionInfo bytes can parse as an empty CursorLogEntry.
+ log.debug("Recovered legacy PositionInfo from unknown CursorLogEntry payload, "
+ + "ledgerId: {}, entryId: {}", lh.getId(), entryId);
+ return CompletableFuture.completedFuture(RecoveryDecision.recovered(RecoveredState.legacy(bytes)));
+ }
+ return FutureUtil.failedFuture(new ManagedLedgerException(
+ "Invalid cursor log entry at ledger " + lh.getId() + " entry " + entryId
+ + ": neither checkpoint nor checkpointChunk"));
+ }
+
+ private CompletableFuture recoverFromChunk(
+ LedgerHandle lh, long entryId, CursorCheckpointChunk chunk) {
+ int partCount = chunk.getPartCount();
+ int partIndex = chunk.getPartIndex();
+ if (partIndex < 0 || partIndex >= partCount) {
+ log.warn("Invalid chunk metadata, ledgerId: {}, entryId: {}, partIndex: {}, partCount: {}",
+ lh.getId(), entryId, partIndex, partCount);
+ return FutureUtil.failedFuture(new ManagedLedgerException(
+ "Invalid chunk metadata at ledger " + lh.getId() + " entry " + entryId
+ + ": partIndex=" + partIndex + ", partCount=" + partCount));
+ }
+ if (partIndex != partCount - 1) {
+ log.debug("Chunk is not last part, scanning back, ledgerId: {}, entryId: {}, partIndex: {}, partCount: {}",
+ lh.getId(), entryId, partIndex, partCount);
+ return CompletableFuture.completedFuture(RecoveryDecision.scanBack());
+ }
+ return assemble(lh, entryId, partCount)
+ .handle((cp, error) -> {
+ if (error != null) {
+ log.warn("Chunk assembly failed, scanning back, ledgerId: {}, entryId: {}, partCount: {}",
+ lh.getId(), entryId, partCount, error);
+ return RecoveryDecision.scanBack();
+ }
+ return RecoveryDecision.recovered(new RecoveredState(cp, entryId));
+ });
+ }
+
+ private CompletableFuture assemble(LedgerHandle lh, long lastPartEntryId, int partCount) {
+ long firstId = lastPartEntryId - partCount + 1;
+ if (firstId < 0) {
+ return FutureUtil.failedFuture(new ManagedLedgerException(
+ "Chunk assembly underflow: first part entry id " + firstId
+ + " (lastPartEntryId=" + lastPartEntryId + ", partCount=" + partCount + ")"));
+ }
+ return readEntryRange(lh, firstId, lastPartEntryId).thenApply(partsBytes -> {
+ if (partsBytes.size() != partCount) {
+ throw new RuntimeException(new ManagedLedgerException(
+ "Chunk assembly mismatch: expected " + partCount + ", got " + partsBytes.size()));
+ }
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ for (int i = 0; i < partCount; i++) {
+ CursorLogEntry part;
+ try {
+ part = CursorLogEntry.parseFrom(partsBytes.get(i));
+ } catch (Exception e) {
+ throw new RuntimeException(new ManagedLedgerException("Failed to parse chunk part " + i, e));
+ }
+ if (!part.hasCheckpointChunk()) {
+ throw new RuntimeException(new ManagedLedgerException("Part " + i + " is not a chunk"));
+ }
+ CursorCheckpointChunk chunk = part.getCheckpointChunk();
+ if (chunk.getPartIndex() != i || chunk.getPartCount() != partCount) {
+ throw new RuntimeException(new ManagedLedgerException(
+ "Part " + i + ": partIndex=" + chunk.getPartIndex()
+ + " partCount=" + chunk.getPartCount() + ", expected " + i + "/" + partCount));
+ }
+ byte[] checkpointBytes = chunk.getCheckpointBytes().toByteArray();
+ bos.write(checkpointBytes, 0, checkpointBytes.length);
+ }
+ try {
+ CursorCheckpoint cp = CursorCheckpoint.parseFrom(bos.toByteArray());
+ log.debug("Assembled chunked checkpoint, ledgerId: {}, partCount: {}, assembledBytes: {}",
+ lh.getId(), partCount, bos.size());
+ return cp;
+ } catch (Exception e) {
+ throw new RuntimeException(new ManagedLedgerException("Failed to parse assembled checkpoint", e));
+ }
+ });
+ }
+
+ private CompletableFuture scanBack(LedgerHandle lh, long fromEntryId) {
+ CompletableFuture hintFuture = null;
+ if (zkCheckpointLedgerId >= 0 && zkCheckpointEntryId >= 0
+ && zkCheckpointLedgerId == lh.getId() && zkCheckpointEntryId < fromEntryId) {
+ log.debug("Trying ZK checkpoint hint before scan-back, ledgerId: {}, hintEntryId: {}, fromEntryId: {}",
+ lh.getId(), zkCheckpointEntryId, fromEntryId);
+ hintFuture = recoverEntryAt(lh, zkCheckpointEntryId, false)
+ .thenCompose(decision -> decision.shouldScanBack
+ ? scanBackStep(lh, fromEntryId - 1)
+ : CompletableFuture.completedFuture(decision.state));
+ }
+ if (hintFuture != null) {
+ return hintFuture.exceptionally(t -> {
+ log.debug("ZK checkpoint hint failed, falling back to sequential scan-back, "
+ + "ledgerId: {}, hintEntryId: {}",
+ lh.getId(), zkCheckpointEntryId);
+ return null;
+ }).thenCompose(state ->
+ state != null ? CompletableFuture.completedFuture(state)
+ : scanBackStep(lh, fromEntryId - 1));
+ }
+ return scanBackStep(lh, fromEntryId - 1);
+ }
+
+ private CompletableFuture scanBackStep(LedgerHandle lh, long entryId) {
+ long floor = lh.getLastAddConfirmed() - MAX_SCAN_BACK;
+ if (entryId < 0 || entryId < floor) {
+ log.warn("Scan-back exhausted without recoverable checkpoint, ledgerId: {}, entryId: {}, floor: {}",
+ lh.getId(), entryId, floor);
+ return FutureUtil.failedFuture(new ManagedLedgerException(
+ "scanBack exhausted without finding a complete checkpoint"));
+ }
+ return recoverEntryAt(lh, entryId, true)
+ .thenCompose(decision -> decision.shouldScanBack
+ ? scanBackStep(lh, entryId - 1)
+ : CompletableFuture.completedFuture(decision.state));
+ }
+
+ private static CompletableFuture addEntry(LedgerHandle lh, byte[] data) {
+ CompletableFuture future = new CompletableFuture<>();
+ lh.asyncAddEntry(data, (rc, handle, entryId, ctx) -> {
+ if (rc == BKException.Code.OK) {
+ future.complete(entryId);
+ } else {
+ future.completeExceptionally(BKException.create(rc));
+ }
+ }, null);
+ return future;
+ }
+
+ private static CompletableFuture readEntry(LedgerHandle lh, long entryId) {
+ CompletableFuture future = new CompletableFuture<>();
+ lh.asyncReadEntries(entryId, entryId, (rc, lh1, entries, ctx) -> {
+ if (rc != BKException.Code.OK) {
+ future.completeExceptionally(BKException.create(rc));
+ return;
+ }
+ if (!entries.hasMoreElements()) {
+ future.complete(null);
+ return;
+ }
+ future.complete(entries.nextElement().getEntry());
+ }, null);
+ return future;
+ }
+
+ private static CompletableFuture> readEntryRange(LedgerHandle lh, long firstId, long lastId) {
+ CompletableFuture> future = new CompletableFuture<>();
+ lh.asyncReadEntries(firstId, lastId, (rc, lh1, entries, ctx) -> {
+ if (rc != BKException.Code.OK) {
+ future.completeExceptionally(BKException.create(rc));
+ return;
+ }
+ List result = new ArrayList<>();
+ while (entries.hasMoreElements()) {
+ result.add(entries.nextElement().getEntry());
+ }
+ future.complete(result);
+ }, null);
+ return future;
+ }
+
+ private static boolean isLegacyPositionInfo(byte[] bytes) {
+ try {
+ PositionInfo.parseFrom(bytes);
+ return true;
+ } catch (Exception e) {
+ return false;
+ }
+ }
+
+ record AppendResult(int totalBytes, long commitEntryId) {}
+
+ private static final class RecoveryDecision {
+ private final RecoveredState state;
+ private final boolean shouldScanBack;
+
+ private RecoveryDecision(RecoveredState state, boolean shouldScanBack) {
+ this.state = state;
+ this.shouldScanBack = shouldScanBack;
+ }
+
+ static RecoveryDecision recovered(RecoveredState state) {
+ return new RecoveryDecision(state, false);
+ }
+
+ static RecoveryDecision scanBack() {
+ return new RecoveryDecision(null, true);
+ }
+ }
+
+ static final class RecoveredState {
+ final CursorCheckpoint checkpoint;
+ final long commitEntryId;
+ final byte[] legacyBytes;
+
+ private RecoveredState(CursorCheckpoint checkpoint, long commitEntryId) {
+ this.checkpoint = checkpoint;
+ this.commitEntryId = commitEntryId;
+ this.legacyBytes = null;
+ }
+
+ private RecoveredState(byte[] legacyBytes) {
+ this.checkpoint = null;
+ this.commitEntryId = -1;
+ this.legacyBytes = legacyBytes;
+ }
+
+ static RecoveredState legacy(byte[] bytes) {
+ return new RecoveredState(bytes);
+ }
+
+ boolean isLegacy() {
+ return legacyBytes != null;
+ }
+ }
+}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointPersistence.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointPersistence.java
new file mode 100644
index 0000000000000..fd677d7dd3de6
--- /dev/null
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointPersistence.java
@@ -0,0 +1,487 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger.impl;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.protobuf.ByteString;
+import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.locks.ReadWriteLock;
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.client.BKException;
+import org.apache.bookkeeper.client.BookKeeper;
+import org.apache.bookkeeper.client.LedgerHandle;
+import org.apache.bookkeeper.mledger.ManagedLedgerException;
+import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.PositionFactory;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats.AckState;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats.AckStateRef;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats.BatchedEntryDeletionIndexInfo;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats.CursorCheckpoint;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats.LongProperty;
+
+@Slf4j
+class CursorCheckpointPersistence {
+
+ private final CursorCheckpointLog writer;
+ private final ReadWriteLock lock;
+ private final BookKeeper bookKeeper;
+ private final BookKeeper.DigestType digestType;
+ private final byte[] password;
+ @Getter
+ private final boolean perLedgerEntryPersistEnabled;
+
+ // Position of each msg ledger's latest checkpoint in the cursor ledger; the AckStateRef
+ // targets are derived from this index.
+ private final Long2ObjectOpenHashMap lastCheckpointPos = new Long2ObjectOpenHashMap<>();
+ private volatile CompletableFuture lastPersist =
+ CompletableFuture.completedFuture(null);
+
+ public CursorCheckpointPersistence(CursorCheckpointLog writer, ReadWriteLock lock,
+ BookKeeper bookKeeper,
+ BookKeeper.DigestType digestType, byte[] password,
+ boolean perLedgerEntryPersistEnabled) {
+ this.writer = writer;
+ this.lock = lock;
+ this.bookKeeper = bookKeeper;
+ this.digestType = digestType;
+ this.password = password;
+ this.perLedgerEntryPersistEnabled = perLedgerEntryPersistEnabled;
+ }
+
+ public void setZkCheckpointHint(long cursorLedgerId, long entryId) {
+ writer.setZkCheckpointHint(cursorLedgerId, entryId);
+ }
+
+ public synchronized CompletableFuture persist(
+ LedgerHandle lh, Position mdPos, Map properties,
+ ManagedCursorImpl cursor) {
+ lastPersist = lastPersist
+ .exceptionally(e -> null)
+ .thenCompose(ignored -> doPersist(lh, mdPos, properties, cursor));
+ return lastPersist;
+ }
+
+ /**
+ * Immutable snapshot of cursor state for one flush. Building reads only from this snapshot,
+ * not live cursor state, so acks arriving during async appends don't wedge the flush.
+ */
+ private static final class PersistContext {
+ final long mdLedgerId;
+ // Ledgers whose checkpoint is written by this flush (dirty + self-healed positionless
+ // active ledgers), excluding the mark-delete ledger.
+ final Set flushedLedgers;
+ // Ledgers that may be referenced by this flush's checkpoints.
+ final Set activeLedgers;
+ final Map dirtyBitmaps;
+ final Map> dirtyBatchAcks;
+ final byte[] mdBitmap;
+ final List mdBatchAcks;
+
+ PersistContext(long mdLedgerId, Set flushedLedgers, Set activeLedgers,
+ Map dirtyBitmaps,
+ Map> dirtyBatchAcks,
+ byte[] mdBitmap, List mdBatchAcks) {
+ this.mdLedgerId = mdLedgerId;
+ this.flushedLedgers = flushedLedgers;
+ this.activeLedgers = activeLedgers;
+ this.dirtyBitmaps = dirtyBitmaps;
+ this.dirtyBatchAcks = dirtyBatchAcks;
+ this.mdBitmap = mdBitmap;
+ this.mdBatchAcks = mdBatchAcks;
+ }
+ }
+
+ private PersistContext createPersistContext(ManagedCursorImpl cursor, Position mdPos) {
+ lock.readLock().lock();
+ try {
+ long mdLedgerId = mdPos.getLedgerId();
+ Set clearedDirtyLedgers = cursor.individualDeletedMessages.snapshotAndClearDirtyLedgers();
+ Set flushedLedgers = new HashSet<>(clearedDirtyLedgers);
+ flushedLedgers.remove(mdLedgerId);
+
+ Set activeLedgers = new HashSet<>();
+ cursor.individualDeletedMessages.forEachActiveLedger(activeLedgers::add);
+ Map> batchAcksByLedger = groupBatchAcksByLedger(cursor);
+ batchAcksByLedger.keySet().forEach(activeLedgers::add);
+
+ // Self-heal: active ledgers without a persisted checkpoint are flushed this round,
+ // otherwise they'd have no position to reference.
+ for (long id : activeLedgers) {
+ if (id != mdLedgerId && !lastCheckpointPos.containsKey(id)) {
+ flushedLedgers.add(id);
+ }
+ }
+
+ Map dirtyBitmaps = new HashMap<>();
+ Map> dirtyBatchAcks = new HashMap<>();
+ for (long id : flushedLedgers) {
+ dirtyBitmaps.put(id, cursor.individualDeletedMessages.bitmapOf(id));
+ dirtyBatchAcks.put(id, batchAcksByLedger.getOrDefault(id, Collections.emptyList()));
+ }
+ byte[] mdBitmap = cursor.individualDeletedMessages.bitmapOf(mdLedgerId);
+ List mdBatchAcks =
+ batchAcksByLedger.getOrDefault(mdLedgerId, Collections.emptyList());
+ return new PersistContext(mdLedgerId, flushedLedgers, activeLedgers, dirtyBitmaps,
+ dirtyBatchAcks, mdBitmap, mdBatchAcks);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private static Map> groupBatchAcksByLedger(
+ ManagedCursorImpl cursor) {
+ Map> result = new HashMap<>();
+ if (cursor.batchDeletedIndexes == null) {
+ return result;
+ }
+ cursor.batchDeletedIndexes.forEach((position, bitSet) -> {
+ List list =
+ result.computeIfAbsent(position.getLedgerId(), k -> new ArrayList<>());
+ BatchedEntryDeletionIndexInfo.Builder infoBuilder = BatchedEntryDeletionIndexInfo.newBuilder();
+ infoBuilder.getPositionBuilder()
+ .setLedgerId(position.getLedgerId())
+ .setEntryId(position.getEntryId());
+ bitSet.stream().forEach(infoBuilder::addDeleteSet);
+ list.add(infoBuilder.build());
+ });
+ return result;
+ }
+
+ private CompletableFuture doPersist(
+ LedgerHandle lh, Position mdPos, Map properties,
+ ManagedCursorImpl cursor) {
+ PersistContext ctx = createPersistContext(cursor, mdPos);
+
+ // Drop positions for ledgers below mark-delete that are no longer active. Active ledgers
+ // are kept: batch-index entries may still be in memory before the align cleanup runs.
+ lock.writeLock().lock();
+ try {
+ lastCheckpointPos.keySet().removeIf(id -> id < ctx.mdLedgerId && !ctx.activeLedgers.contains(id));
+ } finally {
+ lock.writeLock().unlock();
+ }
+
+ List dirtyOrder = new ArrayList<>(ctx.flushedLedgers);
+ Collections.sort(dirtyOrder);
+ // mdLedger is written first so other checkpoints in this flush can reference its position.
+ List writeOrder = new ArrayList<>(dirtyOrder.size() + 1);
+ writeOrder.add(ctx.mdLedgerId);
+ writeOrder.addAll(dirtyOrder);
+
+ final Set appendedLedgers = new HashSet<>();
+ CompletableFuture chain =
+ CompletableFuture.completedFuture(null);
+ for (long ledgerId : writeOrder) {
+ chain = chain.thenCompose(ignored -> {
+ byte[] bitmap = ledgerId == ctx.mdLedgerId ? ctx.mdBitmap : ctx.dirtyBitmaps.get(ledgerId);
+ List batchAcks =
+ ledgerId == ctx.mdLedgerId ? ctx.mdBatchAcks : ctx.dirtyBatchAcks.get(ledgerId);
+ CursorCheckpoint checkpoint = buildCheckpoint(ctx, ledgerId, bitmap, batchAcks,
+ mdPos, properties);
+ return writer.appendCheckpoint(lh, checkpoint).thenApply(result -> {
+ lock.writeLock().lock();
+ try {
+ Position persistedPos = PositionFactory.create(lh.getId(), result.commitEntryId());
+ recordCheckpointPos(ledgerId, persistedPos);
+ appendedLedgers.add(ledgerId);
+ } finally {
+ lock.writeLock().unlock();
+ }
+ return result;
+ });
+ });
+ }
+
+ return chain.exceptionally(error -> {
+ restoreDirtyForFailedLedgers(cursor, ctx.flushedLedgers, appendedLedgers);
+ throw new CompletionException(error);
+ });
+ }
+
+ /**
+ * Re-marks dirty for ledgers whose checkpoint was not appended, so the next flush retries them.
+ */
+ /**
+ * Records the position of a msg ledger's latest checkpoint. Persists are serialized
+ * ({@link #persist} chains behind the previous one), so puts normally arrive in order;
+ * the monotonic guard is defense-in-depth: a stale completion must never regress the
+ * index, otherwise a later AckStateRef would point at an older (smaller) ack bitmap
+ * and the acks recorded in the newer checkpoint would be lost on recovery.
+ */
+ @VisibleForTesting
+ void recordCheckpointPos(long msgLedgerId, Position pos) {
+ Position prev = lastCheckpointPos.get(msgLedgerId);
+ if (prev == null || pos.compareTo(prev) > 0) {
+ lastCheckpointPos.put(msgLedgerId, pos);
+ }
+ }
+
+ @VisibleForTesting
+ Position checkpointPosOf(long msgLedgerId) {
+ return lastCheckpointPos.get(msgLedgerId);
+ }
+
+ private void restoreDirtyForFailedLedgers(ManagedCursorImpl cursor, Set flushedLedgers,
+ Set appendedLedgers) {
+ Set failedLedgers = new HashSet<>(flushedLedgers);
+ failedLedgers.removeAll(appendedLedgers);
+ if (failedLedgers.isEmpty()) {
+ return;
+ }
+ lock.writeLock().lock();
+ try {
+ cursor.individualDeletedMessages.restoreDirtyLedgers(failedLedgers);
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ // ============================ recover ============================
+
+ public CompletableFuture recover(LedgerHandle lh) {
+ return writer.readLatest(lh).thenCompose(state -> {
+ if (state.isLegacy()) {
+ return CompletableFuture.completedFuture(RecoveredCheckpoint.legacy(state.legacyBytes));
+ }
+ CursorCheckpoint cp = state.checkpoint;
+ return fetchAckStateRefs(cp, lh).thenApply(fetched -> {
+ validateRecoveredAckData(cp, fetched);
+ rebuildLastCheckpointPos(lh, state.commitEntryId, cp);
+ return RecoveredCheckpoint.of(cp, fetched);
+ });
+ });
+ }
+
+ /** Validates no duplicate msgLedgerIds and that all refs resolved. Fails fast on corruption. */
+ private static void validateRecoveredAckData(CursorCheckpoint cp, Map fetched) {
+ Set expected = new HashSet<>(cp.getAckStatesCount() + cp.getAckStateRefsCount());
+ for (AckState ackState : cp.getAckStatesList()) {
+ if (!expected.add(ackState.getMsgLedgerId())) {
+ throw new RuntimeException(new ManagedLedgerException(
+ "Recovery inconsistency: duplicate msgLedgerId " + ackState.getMsgLedgerId()
+ + " in inline ack states"));
+ }
+ }
+ for (AckStateRef ref : cp.getAckStateRefsList()) {
+ if (!expected.add(ref.getMsgLedgerId())) {
+ throw new RuntimeException(new ManagedLedgerException(
+ "Recovery inconsistency: msgLedgerId " + ref.getMsgLedgerId()
+ + " appears both inline and in ack state refs"));
+ }
+ }
+ if (!fetched.keySet().equals(expected)) {
+ throw new RuntimeException(new ManagedLedgerException(
+ "Recovery inconsistency: expected ack states for " + expected
+ + ", got " + fetched.keySet()));
+ }
+ }
+
+ /** Rebuilds lastCheckpointPos from the recovered checkpoint so the first persist can emit refs. */
+ private void rebuildLastCheckpointPos(LedgerHandle lh, long commitEntryId, CursorCheckpoint cp) {
+ lock.writeLock().lock();
+ try {
+ lastCheckpointPos.clear();
+ Position inlinePos = PositionFactory.create(lh.getId(), commitEntryId);
+ for (AckState ackState : cp.getAckStatesList()) {
+ lastCheckpointPos.put(ackState.getMsgLedgerId(), inlinePos);
+ }
+ for (AckStateRef ref : cp.getAckStateRefsList()) {
+ lastCheckpointPos.put(ref.getMsgLedgerId(),
+ PositionFactory.create(ref.getCursorLedgerId(), ref.getEntryId()));
+ }
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ public CompletableFuture recoverWithHint(
+ LedgerHandle lh, long hintCursorLedgerId, long hintEntryId) {
+ if (hintCursorLedgerId >= 0 && hintEntryId >= 0) {
+ setZkCheckpointHint(hintCursorLedgerId, hintEntryId);
+ }
+ return recover(lh);
+ }
+
+ private CompletableFuture> fetchAckStateRefs(CursorCheckpoint cp, LedgerHandle lh) {
+ Map result = new HashMap<>();
+ for (AckState ackState : cp.getAckStatesList()) {
+ result.put(ackState.getMsgLedgerId(), toAckStateData(ackState));
+ }
+ List refs = cp.getAckStateRefsList();
+ if (refs.isEmpty()) {
+ return CompletableFuture.completedFuture(result);
+ }
+ List> fetches = new ArrayList<>(refs.size());
+ for (AckStateRef ref : refs) {
+ fetches.add(fetchAckState(ref, lh));
+ }
+ return CompletableFuture.allOf(fetches.toArray(new CompletableFuture[0]))
+ .thenApply(ignored -> {
+ for (int i = 0; i < refs.size(); i++) {
+ result.put(refs.get(i).getMsgLedgerId(), fetches.get(i).join());
+ }
+ return result;
+ });
+ }
+
+ private CompletableFuture fetchAckState(AckStateRef ref, LedgerHandle lh) {
+ return readCheckpointFromLedger(lh, ref.getCursorLedgerId(), ref.getEntryId())
+ .thenApply(cp -> extractAckState(cp, ref));
+ }
+
+ private static AckStateData extractAckState(CursorCheckpoint cp, AckStateRef ref) {
+ for (AckState ackState : cp.getAckStatesList()) {
+ if (ackState.getMsgLedgerId() == ref.getMsgLedgerId()) {
+ return toAckStateData(ackState);
+ }
+ }
+ throw new RuntimeException(new ManagedLedgerException(
+ "AckStateRef target " + ref + " does not contain msgLedgerId "
+ + ref.getMsgLedgerId()));
+ }
+
+ private static AckStateData toAckStateData(AckState ackState) {
+ return new AckStateData(
+ ackState.hasAckBitmap() ? ackState.getAckBitmap().toByteArray() : null,
+ ackState.getBatchAcksList() != null
+ ? ackState.getBatchAcksList() : Collections.emptyList());
+ }
+
+ private CompletableFuture readCheckpointFromLedger(
+ LedgerHandle lh, long ledgerId, long entryId) {
+ CompletableFuture future = new CompletableFuture<>();
+ if (ledgerId == lh.getId()) {
+ // Refs into the recovered cursor ledger itself can reuse the already-open handle.
+ writer.readAt(lh, entryId)
+ .whenComplete((state, error) -> completeCheckpointRead(future, ledgerId, entryId, state, error));
+ return future;
+ }
+ bookKeeper.asyncOpenLedgerNoRecovery(ledgerId, digestType, password, (rc, handle, ctx) -> {
+ if (rc != BKException.Code.OK) {
+ future.completeExceptionally(BKException.create(rc));
+ return;
+ }
+ writer.readAt(handle, entryId).whenComplete((state, error) -> {
+ handle.asyncClose((closeRc, closeHandle, closeCtx) -> {}, null);
+ completeCheckpointRead(future, ledgerId, entryId, state, error);
+ });
+ }, null);
+ return future;
+ }
+
+ private static void completeCheckpointRead(CompletableFuture future, long ledgerId,
+ long entryId, CursorCheckpointLog.RecoveredState state,
+ Throwable error) {
+ if (error != null) {
+ future.completeExceptionally(error);
+ } else if (state.isLegacy()) {
+ future.completeExceptionally(new ManagedLedgerException(
+ "AckStateRef target entry " + entryId + " in ledger " + ledgerId
+ + " is not a checkpoint"));
+ } else {
+ future.complete(state.checkpoint);
+ }
+ }
+
+ // ============================ checkpoint builder ============================
+
+ /**
+ * Builds a checkpoint for one msg ledger: inline ack state plus refs to every other active
+ * ledger's latest position. Refs stay valid because cursor ledgers are only GC'd once
+ * mark-delete passes the last ledger holding acks.
+ */
+ private CursorCheckpoint buildCheckpoint(PersistContext ctx, long msgLedgerId, byte[] bitmap,
+ List batchAcks,
+ Position mdPos, Map properties) {
+ CursorCheckpoint.Builder cpBuilder = CursorCheckpoint.newBuilder()
+ .setMarkDeleteLedgerId(mdPos.getLedgerId())
+ .setMarkDeleteEntryId(mdPos.getEntryId());
+ if (properties != null) {
+ properties.forEach((name, value) -> {
+ cpBuilder.addProperties(LongProperty.newBuilder().setName(name).setValue(value).build());
+ });
+ }
+ addAckState(cpBuilder, msgLedgerId, bitmap, batchAcks);
+ lock.readLock().lock();
+ try {
+ for (long id : ctx.activeLedgers) {
+ if (id != msgLedgerId) {
+ Position pos = lastCheckpointPos.get(id);
+ if (pos == null) {
+ if (ctx.flushedLedgers.contains(id)) {
+ continue;
+ }
+ throw new IllegalStateException(
+ "Missing lastCheckpointPos for active msgLedgerId " + id);
+ }
+ cpBuilder.addAckStateRefs(AckStateRef.newBuilder()
+ .setMsgLedgerId(id)
+ .setCursorLedgerId(pos.getLedgerId())
+ .setEntryId(pos.getEntryId())
+ .build());
+ }
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ return cpBuilder.build();
+ }
+
+ private static void addAckState(CursorCheckpoint.Builder cp, long msgLedgerId, byte[] bitmap,
+ List batchAcks) {
+ AckState.Builder ackStateBuilder = AckState.newBuilder().setMsgLedgerId(msgLedgerId);
+ if (bitmap != null && bitmap.length > 0) {
+ ackStateBuilder.setAckBitmap(ByteString.copyFrom(bitmap));
+ }
+ if (batchAcks != null && !batchAcks.isEmpty()) {
+ ackStateBuilder.addAllBatchAcks(batchAcks);
+ }
+ cp.addAckStates(ackStateBuilder.build());
+ }
+
+ // ============================ DTOs ============================
+
+ record AckStateData(byte[] ackBitmap, List batchAcks) {}
+
+ record RecoveredCheckpoint(CursorCheckpoint checkpoint,
+ Map ackData,
+ byte[] legacyBytes) {
+ static RecoveredCheckpoint of(CursorCheckpoint cp, Map ackData) {
+ return new RecoveredCheckpoint(cp, ackData, null);
+ }
+
+ static RecoveredCheckpoint legacy(byte[] bytes) {
+ return new RecoveredCheckpoint(null, null, bytes);
+ }
+
+ boolean isLegacy() {
+ return legacyBytes != null;
+ }
+ }
+}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryImpl.java
index e0e2b859794b5..87a037a90f60d 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryImpl.java
@@ -94,6 +94,10 @@ public static EntryImpl create(Position position, ByteBuf data) {
return entry;
}
+ public static EntryImpl create(Position position, ByteBuf data, int expectedReadCount) {
+ return create(position, data);
+ }
+
public static EntryImpl create(EntryImpl other) {
EntryImpl entry = RECYCLER.get();
entry.timestamp = System.nanoTime();
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
index 790c81cadc9ba..4105f98923874 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
@@ -26,17 +26,18 @@
import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.DEFAULT_LEDGER_DELETE_RETRIES;
import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.createManagedLedgerException;
import static org.apache.bookkeeper.mledger.util.Errors.isNoSuchLedgerExistsException;
+import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.MoreObjects;
import com.google.common.collect.Collections2;
import com.google.common.collect.Lists;
import com.google.common.collect.Range;
import com.google.common.util.concurrent.RateLimiter;
-import com.google.protobuf.InvalidProtocolBufferException;
import java.time.Clock;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.BitSet;
+import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
@@ -47,19 +48,20 @@
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
-import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicLongFieldUpdater;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Function;
+import java.util.function.IntFunction;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.LongStream;
@@ -95,6 +97,9 @@
import org.apache.bookkeeper.mledger.ScanOutcome;
import org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback;
import org.apache.bookkeeper.mledger.proto.MLDataFormats;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats.AckStateRef;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats.BatchedEntryDeletionIndexInfo;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats.CursorCheckpoint;
import org.apache.bookkeeper.mledger.proto.MLDataFormats.LongListMap;
import org.apache.bookkeeper.mledger.proto.MLDataFormats.LongProperty;
import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo;
@@ -104,6 +109,7 @@
import org.apache.bookkeeper.mledger.proto.MLDataFormats.PositionInfo.Builder;
import org.apache.bookkeeper.mledger.proto.MLDataFormats.StringProperty;
import org.apache.bookkeeper.mledger.util.ManagedLedgerUtils;
+import org.apache.commons.lang3.mutable.MutableBoolean;
import org.apache.commons.lang3.mutable.MutableInt;
import org.apache.commons.lang3.mutable.MutableLong;
import org.apache.commons.lang3.tuple.Pair;
@@ -112,7 +118,6 @@
import org.apache.pulsar.common.util.FutureUtil;
import org.apache.pulsar.common.util.collections.LongPairRangeSet;
import org.apache.pulsar.common.util.collections.LongPairRangeSet.LongPairConsumer;
-import org.apache.pulsar.common.util.collections.LongPairRangeSet.RangeBoundConsumer;
import org.apache.pulsar.metadata.api.Stat;
import org.jspecify.annotations.Nullable;
import org.slf4j.Logger;
@@ -198,11 +203,8 @@ public class ManagedCursorImpl implements ManagedCursor {
private static final LongPairConsumer positionRangeConverter = PositionFactory::create;
- private static final RangeBoundConsumer positionRangeReverseConverter =
- (position) -> new LongPairRangeSet.LongPair(position.getLedgerId(), position.getEntryId());
-
private static final LongPairConsumer recyclePositionRangeConverter = PositionRecyclable::get;
- protected final RangeSetWrapper individualDeletedMessages;
+ protected final PositionRangeSet individualDeletedMessages;
// Maintain the deletion status for batch messages
// (ledgerId, entryId) -> deletion indexes
@@ -211,6 +213,11 @@ public class ManagedCursorImpl implements ManagedCursor {
@Nullable protected final ConcurrentSkipListMap batchDeletedIndexes;
protected final ReadWriteLock lock = new ReentrantReadWriteLock();
+ @VisibleForTesting
+ final CursorCheckpointPersistence ackPersistence;
+ private volatile long pendingCheckpointHintLedgerId = -1;
+ private volatile long pendingCheckpointHintEntryId = -1;
+ private final Set allCursorLedgerIds = ConcurrentHashMap.newKeySet();
private RateLimiter markDeleteLimiter;
// The cursor is considered "dirty" when there are mark-delete updates that are only applied in memory,
// because of the rate limiting.
@@ -223,6 +230,7 @@ public class ManagedCursorImpl implements ManagedCursor {
private long entriesReadCount;
private long entriesReadSize;
private int individualDeletedMessagesSerializedSize;
+ private final AtomicBoolean lastCursorDataFullyPersistable = new AtomicBoolean(true);
private static final String COMPACTION_CURSOR_NAME = "__compaction";
private volatile boolean cacheReadEntry = false;
@@ -240,6 +248,13 @@ class MarkDeleteEntry {
final Object ctx;
final Map properties;
final Runnable alignAcknowledgeStatusAfterPersisted;
+ // True for custom aligns (e.g. resetCursor) that must run even when persist is deferred.
+ // Entries that carry a custom align runnable (cursor reset) must surface persist
+ // failures to their caller instead of silently reporting success: the in-memory
+ // state was NOT applied, so a "successful" reset would be a silent no-op.
+ final boolean propagatePersistFailure;
+ // Set once the position is actually durable; gates the default align.
+ volatile boolean persistedSuccessfully;
// If the callbackGroup is set, it means this mark-delete request was done on behalf of a group of request (just
// persist the last one in the chain). In this case we need to trigger the callbacks for every request in the
@@ -248,11 +263,17 @@ class MarkDeleteEntry {
public MarkDeleteEntry(Position newPosition, Map properties,
MarkDeleteCallback callback, Object ctx) {
- this(newPosition, properties, callback, ctx, null);
+ this(newPosition, properties, callback, ctx, null, false);
}
public MarkDeleteEntry(Position newPosition, Map properties,
MarkDeleteCallback callback, Object ctx, Runnable alignAcknowledgeStatusAfterPersisted) {
+ this(newPosition, properties, callback, ctx, alignAcknowledgeStatusAfterPersisted, false);
+ }
+
+ public MarkDeleteEntry(Position newPosition, Map properties,
+ MarkDeleteCallback callback, Object ctx, Runnable alignAcknowledgeStatusAfterPersisted,
+ boolean propagatePersistFailure) {
if (alignAcknowledgeStatusAfterPersisted == null) {
alignAcknowledgeStatusAfterPersisted = () -> {
if (batchDeletedIndexes != null) {
@@ -268,6 +289,7 @@ public MarkDeleteEntry(Position newPosition, Map properties,
this.callback = callback;
this.ctx = ctx;
this.alignAcknowledgeStatusAfterPersisted = alignAcknowledgeStatusAfterPersisted;
+ this.propagatePersistFailure = propagatePersistFailure;
}
public void triggerComplete() {
@@ -365,14 +387,19 @@ protected ManagedCursorImpl(BookKeeper bookkeeper, ManagedLedgerImpl ledger, Str
this.cursorProperties = Collections.emptyMap();
this.ledger = ledger;
this.name = cursorName;
- this.individualDeletedMessages = new RangeSetWrapper<>(positionRangeConverter,
- positionRangeReverseConverter, this);
+ this.individualDeletedMessages = new PositionRangeSet(positionRangeConverter,
+ getConfig().isPersistentUnackedRangesWithPerLedgerEntryEnabled());
if (getConfig().isDeletionAtBatchIndexLevelEnabled()) {
this.batchDeletedIndexes = new ConcurrentSkipListMap<>();
} else {
this.batchDeletedIndexes = null;
}
this.digestType = BookKeeper.DigestType.fromApiDigestType(getConfig().getDigestType());
+ CursorCheckpointLog writer = new CursorCheckpointLog(getConfig().getPersistentUnackedRangesMaxEntrySize());
+ this.ackPersistence = new CursorCheckpointPersistence(
+ writer, lock, bookkeeper,
+ this.digestType, getConfig().getPassword(),
+ getConfig().isPersistentUnackedRangesWithPerLedgerEntryEnabled());
PENDING_MARK_DELETED_SUBMITTED_COUNT_UPDATER.set(this, 0);
PENDING_READ_OPS_UPDATER.set(this, 0);
RESET_CURSOR_IN_PROGRESS_UPDATER.set(this, FALSE);
@@ -401,6 +428,35 @@ public Map getProperties() {
return lastMarkDeleteEntry != null ? lastMarkDeleteEntry.properties : Collections.emptyMap();
}
+ private void recordAckStats(long latencyMillis) {
+ mbean.recordAck(latencyMillis);
+ var stats = otelCursorStatsOrNull();
+ if (stats != null) {
+ stats.recordAckLatency(this, latencyMillis / 1000.0);
+ }
+ }
+
+ private void recordPersistStats(long latencyMillis) {
+ mbean.recordPersist(latencyMillis);
+ var stats = otelCursorStatsOrNull();
+ if (stats != null) {
+ stats.recordPersistLatency(this, latencyMillis / 1000.0);
+ }
+ }
+
+ private void recordRecoverStats(long latencyMillis, boolean success) {
+ mbean.recordRecover(latencyMillis, success);
+ var stats = otelCursorStatsOrNull();
+ if (stats != null) {
+ stats.recordRecoverLatency(this, latencyMillis / 1000.0);
+ }
+ }
+
+ // Null when the ledger (and thus its factory) is mocked in unit tests.
+ private OpenTelemetryManagedCursorStats otelCursorStatsOrNull() {
+ return ledger.getFactory() == null ? null : ledger.getFactory().getOpenTelemetryManagedCursorStats();
+ }
+
@Override
public boolean isCursorDataFullyPersistable() {
lock.readLock().lock();
@@ -546,6 +602,22 @@ public boolean removeProperty(String key) {
void recover(final VoidCallback callback) {
// Read the meta-data ledgerId from the store
log.info("[{}] Recovering from bookkeeper ledger cursor: {}", ledger.getName(), name);
+ final long recoverStartTimeNanos = System.nanoTime();
+ final VoidCallback recoverCallback = new VoidCallback() {
+ @Override
+ public void operationComplete() {
+ recordRecoverStats(
+ TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - recoverStartTimeNanos), true);
+ callback.operationComplete();
+ }
+
+ @Override
+ public void operationFailed(ManagedLedgerException exception) {
+ recordRecoverStats(
+ TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - recoverStartTimeNanos), false);
+ callback.operationFailed(exception);
+ }
+ };
ledger.getStore().asyncGetCursorInfo(ledger.getName(), name, new MetaStoreCallback() {
@Override
public void operationComplete(ManagedCursorInfo info, Stat stat) {
@@ -586,18 +658,18 @@ public void operationComplete(ManagedCursorInfo info, Stat stat) {
}
recoveredCursor(recoveredPosition, recoveredProperties, recoveredCursorProperties, null);
- callback.operationComplete();
+ recoverCallback.operationComplete();
} else {
// Need to proceed and read the last entry in the specified ledger to find out the last position
log.info("[{}] Cursor {} meta-data recover from ledger {}", ledger.getName(), name,
info.getCursorsLedgerId());
- recoverFromLedger(info, callback);
+ recoverFromLedger(info, recoverCallback);
}
}
@Override
public void operationFailed(MetaStoreException e) {
- callback.operationFailed(e);
+ recoverCallback.operationFailed(e);
}
});
}
@@ -655,32 +727,7 @@ protected void recoverFromLedger(final ManagedCursorInfo info, final VoidCallbac
LedgerEntry entry = seq.nextElement();
mbean.addReadCursorLedgerSize(entry.getLength());
- PositionInfo positionInfo;
- try {
- positionInfo = PositionInfo.parseFrom(entry.getEntry());
- } catch (InvalidProtocolBufferException e) {
- callback.operationFailed(new ManagedLedgerException(e));
- return;
- }
-
- Map recoveredProperties = Collections.emptyMap();
- if (positionInfo.getPropertiesCount() > 0) {
- // Recover properties map
- recoveredProperties = new HashMap<>();
- for (int i = 0; i < positionInfo.getPropertiesCount(); i++) {
- LongProperty property = positionInfo.getProperties(i);
- recoveredProperties.put(property.getName(), property.getValue());
- }
- }
-
- Position position = PositionFactory.create(positionInfo.getLedgerId(), positionInfo.getEntryId());
- recoverIndividualDeletedMessages(positionInfo);
- if (getConfig().isDeletionAtBatchIndexLevelEnabled()
- && positionInfo.getBatchedEntryDeletionIndexInfoCount() > 0) {
- recoverBatchDeletedIndexes(positionInfo.getBatchedEntryDeletionIndexInfoList());
- }
- recoveredCursor(position, recoveredProperties, cursorProperties, lh);
- callback.operationComplete();
+ recoverWithCheckpointFormat(info, lh, ledgerId, entry.getEntry(), callback);
}, null);
};
try {
@@ -693,6 +740,27 @@ protected void recoverFromLedger(final ManagedCursorInfo info, final VoidCallbac
}
}
+ private void recoverWithCheckpointFormat(ManagedCursorInfo info, LedgerHandle lh, long cursorLedgerId,
+ byte[] lastEntryBytes, VoidCallback callback) {
+ long hintLedgerId = info.hasLastCmCursorLedgerId() ? info.getLastCmCursorLedgerId() : -1;
+ long hintEntryId = info.hasLastCmCursorLedgerId() ? info.getLastCmEntryId() : -1;
+ ackPersistence.recoverWithHint(lh, hintLedgerId, hintEntryId).whenComplete((recovered, recoverError) -> {
+ if (recoverError != null) {
+ Throwable cause = FutureUtil.unwrapCompletionException(recoverError);
+ log.warn("Per-msgLedger checkpoint recovery failed, rewinding to ZK snapshot, ledgerId: {}",
+ cursorLedgerId, cause);
+ initialize(getRollbackPosition(info), Collections.emptyMap(), cursorProperties, callback);
+ return;
+ }
+ if (recovered != null && !recovered.isLegacy()) {
+ applyRecoveredCheckpoint(recovered, info, lh, cursorProperties, callback);
+ return;
+ }
+ byte[] legacyBytes = recovered != null ? recovered.legacyBytes() : lastEntryBytes;
+ applyLegacyPositionInfo(legacyBytes, info, lh, cursorProperties, callback);
+ });
+ }
+
public void recoverIndividualDeletedMessages(PositionInfo positionInfo) {
if (positionInfo.getIndividualDeletedMessagesCount() > 0) {
recoverIndividualDeletedMessages(positionInfo.getIndividualDeletedMessagesList());
@@ -702,22 +770,7 @@ public void recoverIndividualDeletedMessages(PositionInfo positionInfo) {
try {
Map rangeMap = rangeList.stream().collect(Collectors.toMap(LongListMap::getKey,
list -> list.getValuesList().stream().mapToLong(i -> i).toArray()));
- // Guarantee compatability for the config "unackedRangesOpenCacheSetEnabled".
- if (getConfig().isUnackedRangesOpenCacheSetEnabled()) {
- individualDeletedMessages.build(rangeMap);
- } else {
- RangeSetWrapper rangeSetWrapperV2 = new RangeSetWrapper<>(positionRangeConverter,
- positionRangeReverseConverter, true,
- getConfig().isPersistentUnackedRangesWithMultipleEntriesEnabled());
- rangeSetWrapperV2.build(rangeMap);
- rangeSetWrapperV2.forEach(range -> {
- individualDeletedMessages.addOpenClosed(range.lowerEndpoint().getLedgerId(),
- range.lowerEndpoint().getEntryId(), range.upperEndpoint().getLedgerId(),
- range.upperEndpoint().getEntryId());
- return true;
- });
- rangeSetWrapperV2.clear();
- }
+ individualDeletedMessages.build(rangeMap);
} catch (Exception e) {
log.warn("[{}]-{} Failed to recover individualDeletedMessages from serialized data", ledger.getName(),
name, e);
@@ -751,10 +804,16 @@ private List buildLongPropertiesMap(Map properties) {
}
private void recoverIndividualDeletedMessages(List individualDeletedMessagesList) {
+ recoverIndividualDeletedMessages(individualDeletedMessagesList.size(), individualDeletedMessagesList::get);
+ }
+
+ @VisibleForTesting
+ void recoverIndividualDeletedMessages(int count, IntFunction accessor) {
lock.writeLock().lock();
try {
individualDeletedMessages.clear();
- individualDeletedMessagesList.forEach(messageRange -> {
+ for (int i = 0; i < count; i++) {
+ MessageRange messageRange = accessor.apply(i);
MLDataFormats.NestedPositionInfo lowerEndpoint = messageRange.getLowerEndpoint();
MLDataFormats.NestedPositionInfo upperEndpoint = messageRange.getUpperEndpoint();
@@ -781,7 +840,7 @@ private void recoverIndividualDeletedMessages(List i
individualDeletedMessages.addOpenClosed(upperEndpoint.getLedgerId(), -1,
upperEndpoint.getLedgerId(), upperEndpoint.getEntryId());
}
- });
+ }
} finally {
lock.writeLock().unlock();
}
@@ -809,6 +868,100 @@ private void recoverBatchDeletedIndexes (
}
}
+ /**
+ * Applies a recovered {@link CursorCheckpointPersistence.RecoveredCheckpoint} to
+ * the cursor's in-memory state, then delegates to {@link #recoveredCursor} for finalization.
+ */
+ private void applyRecoveredCheckpoint(CursorCheckpointPersistence.RecoveredCheckpoint recovered,
+ ManagedCursorInfo info, LedgerHandle lh,
+ Map cursorProperties,
+ VoidCallback callback) {
+ CursorCheckpoint cp = recovered.checkpoint();
+ Map recoveredProperties = Collections.emptyMap();
+ if (cp.getPropertiesCount() > 0) {
+ recoveredProperties = new HashMap<>();
+ for (int i = 0; i < cp.getPropertiesCount(); i++) {
+ LongProperty prop = cp.getProperties(i);
+ recoveredProperties.put(prop.getName(), prop.getValue());
+ }
+ }
+ Position position = PositionFactory.create(cp.getMarkDeleteLedgerId(), cp.getMarkDeleteEntryId());
+
+ // Track the recovered cursor ledger and every referenced old cursor ledger so GC can
+ // reclaim them across restarts once mark-delete passes the last ledger holding acks.
+ allCursorLedgerIds.add(lh.getId());
+ for (AckStateRef ref : cp.getAckStateRefsList()) {
+ allCursorLedgerIds.add(ref.getCursorLedgerId());
+ }
+
+ // Apply per-ledger ack bitmaps via bulk build (RoaringBitmap bytes → LongBitmap).
+ Map bitmaps = new HashMap<>();
+ for (Map.Entry e : recovered.ackData().entrySet()) {
+ byte[] bm = e.getValue().ackBitmap();
+ if (bm != null && bm.length > 0) {
+ bitmaps.put(e.getKey(), bm);
+ }
+ }
+ lock.writeLock().lock();
+ try {
+ individualDeletedMessages.buildFromBitmaps(bitmaps);
+ // Apply batch acks (across all ledgers)
+ if (getConfig().isDeletionAtBatchIndexLevelEnabled() && batchDeletedIndexes != null) {
+ for (CursorCheckpointPersistence.AckStateData data : recovered.ackData().values()) {
+ if (data.batchAcks() != null) {
+ for (BatchedEntryDeletionIndexInfo batchInfo : data.batchAcks()) {
+ Position batchPos = PositionFactory.create(
+ batchInfo.getPosition().getLedgerId(), batchInfo.getPosition().getEntryId());
+ BitSet bitSet = new BitSet();
+ for (int i = 0; i < batchInfo.getDeleteSetCount(); i++) {
+ bitSet.set((int) batchInfo.getDeleteSet(i));
+ }
+ batchDeletedIndexes.put(batchPos, bitSet);
+ }
+ }
+ }
+ }
+ // Dirty state is fresh after recovery — nothing to persist until next ack.
+ individualDeletedMessages.resetDirtyKeys();
+ } finally {
+ lock.writeLock().unlock();
+ }
+
+ recoveredCursor(position, recoveredProperties, cursorProperties, lh);
+ callback.operationComplete();
+ }
+
+ /**
+ * Legacy fallback: parses bytes as {@link PositionInfo} and runs the existing
+ * recovery path.
+ */
+ private void applyLegacyPositionInfo(byte[] bytes, ManagedCursorInfo info, LedgerHandle lh,
+ Map cursorProperties, VoidCallback callback) {
+ PositionInfo positionInfo;
+ try {
+ positionInfo = PositionInfo.parseFrom(bytes);
+ } catch (Exception e) {
+ callback.operationFailed(new ManagedLedgerException(e));
+ return;
+ }
+ Map recoveredProperties = Collections.emptyMap();
+ if (positionInfo.getPropertiesCount() > 0) {
+ recoveredProperties = new HashMap<>();
+ for (int i = 0; i < positionInfo.getPropertiesCount(); i++) {
+ LongProperty property = positionInfo.getProperties(i);
+ recoveredProperties.put(property.getName(), property.getValue());
+ }
+ }
+ Position position = PositionFactory.create(positionInfo.getLedgerId(), positionInfo.getEntryId());
+ recoverIndividualDeletedMessages(positionInfo);
+ if (getConfig().isDeletionAtBatchIndexLevelEnabled()
+ && positionInfo.getBatchedEntryDeletionIndexInfoCount() > 0) {
+ recoverBatchDeletedIndexes(positionInfo.getBatchedEntryDeletionIndexInfoList());
+ }
+ recoveredCursor(position, recoveredProperties, cursorProperties, lh);
+ callback.operationComplete();
+ }
+
private void recoveredCursor(Position position, Map properties,
Map cursorProperties,
LedgerHandle recoveredFromCursorLedger) {
@@ -1289,8 +1442,41 @@ public int getNonContiguousDeletedMessagesRangeSerializedSize() {
return this.individualDeletedMessagesSerializedSize;
}
+ long getNumberOfIndividualDeletedMessages() {
+ return individualDeletedMessages.totalCardinality();
+ }
+
+ String getFirstIndividualDeletedMessage() {
+ lock.readLock().lock();
+ try {
+ Range first = individualDeletedMessages.firstRange();
+ if (first == null) {
+ return null;
+ }
+ return PositionFactory.create(first.lowerEndpoint().getLedgerId(),
+ first.lowerEndpoint().getEntryId() + 1).toString();
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
@Override
public long getEstimatedSizeSinceMarkDeletePosition() {
+ Position markDeletePosition = this.markDeletePosition;
+ Position lastPosition = ledger.getLastPosition();
+ if (markDeletePosition == null || markDeletePosition.compareTo(lastPosition) == 0) {
+ return 0;
+ }
+ if (markDeletePosition.compareTo(lastPosition) > 0) {
+ if (!ledger.ledgerExists(lastPosition.getLedgerId())
+ || isMarkDeletePositionOnEmptyCurrentLedger(markDeletePosition)) {
+ return 0;
+ }
+ throw new IllegalArgumentException(String.format(
+ "Cursor %s mark-delete position %s is ahead of the last position %s for managed ledger %s",
+ name, markDeletePosition, lastPosition, ledger.getName()));
+ }
+
long totalSize = ledger.estimateBacklogFromPosition(markDeletePosition);
// Need to subtract size of individual deleted messages
@@ -1303,24 +1489,10 @@ public long getEstimatedSizeSinceMarkDeletePosition() {
long deletedCount = 0;
lock.readLock().lock();
try {
- Range backlogRange = Range.openClosed(markDeletePosition, ledger.getLastPosition());
-
- if (getConfig().isUnackedRangesOpenCacheSetEnabled()) {
- deletedCount = individualDeletedMessages.cardinality(
- backlogRange.lowerEndpoint().getLedgerId(), backlogRange.lowerEndpoint().getEntryId(),
- backlogRange.upperEndpoint().getLedgerId(), backlogRange.upperEndpoint().getEntryId());
- } else {
- AtomicLong deletedCounter = new AtomicLong(0);
- individualDeletedMessages.forEach((r) -> {
- if (r.isConnected(backlogRange)) {
- Range intersection = r.intersection(backlogRange);
- long countInRange = ledger.getNumberOfEntries(intersection);
- deletedCounter.addAndGet(countInRange);
- }
- return true;
- }, recyclePositionRangeConverter);
- deletedCount = deletedCounter.get();
- }
+ Range backlogRange = Range.openClosed(markDeletePosition, lastPosition);
+ deletedCount = individualDeletedMessages.cardinality(
+ backlogRange.lowerEndpoint().getLedgerId(), backlogRange.lowerEndpoint().getEntryId(),
+ backlogRange.upperEndpoint().getLedgerId(), backlogRange.upperEndpoint().getEntryId());
} finally {
lock.readLock().unlock();
}
@@ -1330,7 +1502,7 @@ public long getEstimatedSizeSinceMarkDeletePosition() {
}
// Estimate size by using average entry size from the backlog range
- Range backlogRange = Range.openClosed(markDeletePosition, ledger.getLastPosition());
+ Range backlogRange = Range.openClosed(markDeletePosition, lastPosition);
long totalEntriesInBacklog = ledger.getNumberOfEntries(backlogRange);
if (totalEntriesInBacklog <= deletedCount || totalEntriesInBacklog == 0) {
@@ -1356,6 +1528,12 @@ public long getEstimatedSizeSinceMarkDeletePosition() {
return adjustedSize;
}
+ private boolean isMarkDeletePositionOnEmptyCurrentLedger(Position markDeletePosition) {
+ return ledger.currentLedger != null
+ && markDeletePosition.getLedgerId() == ledger.currentLedger.getId()
+ && ledger.currentLedgerEntries == 0;
+ }
+
private long getNumberOfEntriesInBacklog() {
if (markDeletePosition.compareTo(ledger.getLastPosition()) >= 0) {
return 0;
@@ -1383,6 +1561,50 @@ public long getNumberOfEntriesInBacklog(boolean isPrecise) {
return backlog;
}
+ @Override
+ public boolean hasBacklog() {
+ Position markDeletePosition = this.markDeletePosition;
+ Position lastPosition = ledger.getLastPosition();
+ if (markDeletePosition == null || markDeletePosition.compareTo(lastPosition) >= 0) {
+ return false;
+ }
+
+ Position nextPosition = ledger.getNextValidPosition(markDeletePosition);
+ if (nextPosition.compareTo(lastPosition) > 0) {
+ return false;
+ }
+
+ lock.readLock().lock();
+ try {
+ while (nextPosition.compareTo(lastPosition) <= 0) {
+ Range deletedRange = individualDeletedMessages.rangeContaining(
+ nextPosition.getLedgerId(), nextPosition.getEntryId());
+ if (deletedRange == null) {
+ return true;
+ }
+
+ Position upperEndpoint = deletedRange.upperEndpoint();
+ if (upperEndpoint.compareTo(lastPosition) >= 0) {
+ return false;
+ }
+ nextPosition = ledger.getNextValidPosition(upperEndpoint);
+ }
+ return false;
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public boolean hasBacklog(boolean isPrecise) {
+ if (isPrecise) {
+ return hasBacklog();
+ }
+
+ long backlog = ManagedLedgerImpl.ENTRIES_ADDED_COUNTER_UPDATER.get(ledger) - messagesConsumedCounter;
+ return backlog >= 0 ? backlog > 0 : hasBacklog();
+ }
+
public long getNumberOfEntriesInStorage() {
return ledger.getNumberOfEntries(Range.openClosed(markDeletePosition, ledger.getLastPosition()));
}
@@ -1607,6 +1829,8 @@ protected void internalResetCursor(Position proposedReadPosition,
MSG_CONSUMED_COUNTER_UPDATER.addAndGet(ManagedCursorImpl.this,
-ackedEntriesAfterMdPosition.get().longValue());
markDeletePosition = newMarkDeletePosition;
+ // Clear: a racing mark-delete may have advanced this past the reset position.
+ persistentMarkDeletePosition = null;
lastMarkDeleteEntry = new MarkDeleteEntry(newMarkDeletePosition, isCompactionCursor()
? getProperties() : Collections.emptyMap(), null, null);
individualDeletedMessages.clear();
@@ -1616,6 +1840,7 @@ protected void internalResetCursor(Position proposedReadPosition,
long[] resetWords = ackSetState.getAckSet();
if (resetWords != null) {
batchDeletedIndexes.put(newReadPosition, BitSet.valueOf(resetWords));
+ individualDeletedMessages.markDirtyLedger(newReadPosition.getLedgerId());
}
});
}
@@ -1669,7 +1894,7 @@ public void operationFailed(ManagedLedgerException exception) {
persistentMarkDeletePosition = null;
inProgressMarkDeletePersistPosition = null;
- internalAsyncMarkDelete(newMarkDeletePosition, isCompactionCursor() ? getProperties() : Collections.emptyMap(),
+ internalAsyncMarkDelete(newMarkDeletePosition, isCompactionCursor() ? null : Collections.emptyMap(),
new MarkDeleteCallback() {
@Override
public void markDeleteComplete(Object ctx) {
@@ -1872,45 +2097,22 @@ protected long getNumberOfEntries(Range range) {
log.debug("[{}] getNumberOfEntries. {} allEntries: {}", ledger.getName(), range, allEntries);
}
- AtomicLong deletedEntries = new AtomicLong(0);
+ long deletedEntriesCount = 0;
lock.readLock().lock();
try {
- if (getConfig().isUnackedRangesOpenCacheSetEnabled()) {
- int cardinality = individualDeletedMessages.cardinality(
- range.lowerEndpoint().getLedgerId(), range.lowerEndpoint().getEntryId(),
- range.upperEndpoint().getLedgerId(), range.upperEndpoint().getEntryId());
- deletedEntries.addAndGet(cardinality);
- } else {
- individualDeletedMessages.forEach((r) -> {
- try {
- if (r.isConnected(range)) {
- Range commonEntries = r.intersection(range);
- long commonCount = ledger.getNumberOfEntries(commonEntries);
- if (log.isDebugEnabled()) {
- log.debug("[{}] [{}] Discounting {} entries for already deleted range {}",
- ledger.getName(), name, commonCount, commonEntries);
- }
- deletedEntries.addAndGet(commonCount);
- }
- return true;
- } finally {
- if (r.lowerEndpoint() instanceof PositionRecyclable) {
- ((PositionRecyclable) r.lowerEndpoint()).recycle();
- ((PositionRecyclable) r.upperEndpoint()).recycle();
- }
- }
- }, recyclePositionRangeConverter);
- }
+ deletedEntriesCount = individualDeletedMessages.cardinality(
+ range.lowerEndpoint().getLedgerId(), range.lowerEndpoint().getEntryId(),
+ range.upperEndpoint().getLedgerId(), range.upperEndpoint().getEntryId());
} finally {
lock.readLock().unlock();
}
if (log.isDebugEnabled()) {
- log.debug("[{}] Found {} entries - deleted: {}", ledger.getName(), allEntries - deletedEntries.get(),
- deletedEntries);
+ log.debug("[{}] Found {} entries - deleted: {}", ledger.getName(), allEntries - deletedEntriesCount,
+ deletedEntriesCount);
}
- return allEntries - deletedEntries.get();
+ return allEntries - deletedEntriesCount;
}
@@ -2229,9 +2431,23 @@ public MarkDeletingMarkedPosition(String s) {
public void asyncMarkDelete(final Position position, Map properties,
final MarkDeleteCallback callback, final Object ctx) {
requireNonNull(position);
+ final long ackStartTimeNanos = System.nanoTime();
+ final MarkDeleteCallback ackCallback = new MarkDeleteCallback() {
+ @Override
+ public void markDeleteComplete(Object ctx) {
+ recordAckStats(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - ackStartTimeNanos));
+ callback.markDeleteComplete(ctx);
+ }
+
+ @Override
+ public void markDeleteFailed(ManagedLedgerException exception, Object ctx) {
+ recordAckStats(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - ackStartTimeNanos));
+ callback.markDeleteFailed(exception, ctx);
+ }
+ };
if (isClosed()) {
- callback.markDeleteFailed(new ManagedLedgerException
+ ackCallback.markDeleteFailed(new ManagedLedgerException
.CursorAlreadyClosedException("Cursor was already closed"), ctx);
return;
}
@@ -2241,7 +2457,7 @@ public void asyncMarkDelete(final Position position, Map propertie
log.debug("[{}] cursor reset in progress - ignoring mark delete on position [{}] for cursor [{}]",
ledger.getName(), position, name);
}
- callback.markDeleteFailed(
+ ackCallback.markDeleteFailed(
new ManagedLedgerException("Reset cursor in progress - unable to mark delete position "
+ position.toString()),
ctx);
@@ -2275,7 +2491,7 @@ public void asyncMarkDelete(final Position position, Map propertie
log.debug("[{}] Failed mark delete due to invalid markDelete {} is ahead of last-confirmed-entry {}"
+ " for cursor [{}]", ledger.getName(), position, lastConfirmedEntry, name);
}
- callback.markDeleteFailed(new ManagedLedgerException("Invalid mark deleted position"), ctx);
+ ackCallback.markDeleteFailed(new ManagedLedgerException("Invalid mark deleted position"), ctx);
return;
}
}
@@ -2284,7 +2500,7 @@ public void asyncMarkDelete(final Position position, Map propertie
try {
newPosition = setAcknowledgedPosition(newPosition);
} catch (IllegalArgumentException e) {
- callback.markDeleteFailed(getManagedLedgerException(e), ctx);
+ ackCallback.markDeleteFailed(getManagedLedgerException(e), ctx);
return;
} finally {
lock.writeLock().unlock();
@@ -2294,10 +2510,10 @@ public void asyncMarkDelete(final Position position, Map propertie
if (markDeleteLimiter != null && !markDeleteLimiter.tryAcquire()) {
isDirty = true;
updateLastMarkDeleteEntryToLatest(newPosition, properties);
- callback.markDeleteComplete(ctx);
+ ackCallback.markDeleteComplete(ctx);
return;
}
- internalAsyncMarkDelete(newPosition, properties, callback, ctx, null);
+ internalAsyncMarkDelete(newPosition, properties, ackCallback, ctx, null);
}
private Position ackBatchPosition(Position position) {
@@ -2336,7 +2552,7 @@ protected void internalAsyncMarkDelete(final Position newPosition, Map propertiesToUse =
properties != null ? properties : (last != null ? last.properties : getProperties());
MarkDeleteEntry mdEntry = new MarkDeleteEntry(newPosition, propertiesToUse, callback, ctx,
- alignAcknowledgeStatusAfterPersisted);
+ alignAcknowledgeStatusAfterPersisted, alignAcknowledgeStatusAfterPersisted != null);
// The state might have changed while we were waiting on the queue mutex
switch (state) {
@@ -2433,7 +2649,9 @@ public void operationComplete() {
// point.
lock.writeLock().lock();
try {
- mdEntry.alignAcknowledgeStatus();
+ if (mdEntry.persistedSuccessfully) {
+ mdEntry.alignAcknowledgeStatus();
+ }
} finally {
lock.writeLock().unlock();
}
@@ -2537,8 +2755,38 @@ public void deleteFailed(ManagedLedgerException exception, Object ctx) {
@Override
public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallback callback, Object ctx) {
+ Iterable ackPositions = positions;
+ if (!(ackPositions instanceof Collection)) {
+ List materialized = new ArrayList<>();
+ for (Position position : ackPositions) {
+ materialized.add(position);
+ }
+ ackPositions = materialized;
+ }
+ final int ackPositionCount = ((Collection) ackPositions).size();
+ final long ackStartTimeNanos = System.nanoTime();
+ final AsyncCallbacks.DeleteCallback ackCallback = new AsyncCallbacks.DeleteCallback() {
+ @Override
+ public void deleteComplete(Object ctx) {
+ long latencyMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - ackStartTimeNanos);
+ for (int i = 0; i < ackPositionCount; i++) {
+ recordAckStats(latencyMillis);
+ }
+ callback.deleteComplete(ctx);
+ }
+
+ @Override
+ public void deleteFailed(ManagedLedgerException exception, Object ctx) {
+ long latencyMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - ackStartTimeNanos);
+ for (int i = 0; i < ackPositionCount; i++) {
+ recordAckStats(latencyMillis);
+ }
+ callback.deleteFailed(exception, ctx);
+ }
+ };
+
if (isClosed()) {
- callback.deleteFailed(new ManagedLedgerException
+ ackCallback.deleteFailed(new ManagedLedgerException
.CursorAlreadyClosedException("Cursor was already closed"), ctx);
return;
}
@@ -2553,7 +2801,7 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb
ledger.getName(), name, positions, individualDeletedMessages, markDeletePosition);
}
- for (Position pos : positions) {
+ for (Position pos : ackPositions) {
Position position = requireNonNull(pos);
if (ledger.getLastConfirmedEntry().compareTo(position) < 0) {
if (log.isDebugEnabled()) {
@@ -2561,7 +2809,7 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb
"[{}] Failed mark delete due to invalid markDelete {} is ahead of last-confirmed-entry {} "
+ "for cursor [{}]", ledger.getName(), position, ledger.getLastConfirmedEntry(), name);
}
- callback.deleteFailed(new ManagedLedgerException("Invalid mark deleted position"), ctx);
+ ackCallback.deleteFailed(new ManagedLedgerException("Invalid mark deleted position"), ctx);
return;
}
@@ -2610,6 +2858,10 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb
position.getLedgerId(), position.getEntryId());
MSG_CONSUMED_COUNTER_UPDATER.incrementAndGet(this);
batchDeletedIndexes.remove(position);
+ } else {
+ // Batch-index deletions are not reflected in the individual range bitmap, so
+ // mark the ledger dirty explicitly to ensure they are persisted.
+ individualDeletedMessages.markDirtyLedger(position.getLedgerId());
}
}
}
@@ -2658,12 +2910,12 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb
} catch (Exception e) {
log.warn("[{}] [{}] Error while updating individualDeletedMessages [{}]", ledger.getName(), name,
e.getMessage(), e);
- callback.deleteFailed(getManagedLedgerException(e), ctx);
+ ackCallback.deleteFailed(getManagedLedgerException(e), ctx);
return;
} finally {
lock.writeLock().unlock();
if (skipMarkDeleteBecauseAckedNothing) {
- callback.deleteComplete(ctx);
+ ackCallback.deleteComplete(ctx);
}
}
@@ -2671,7 +2923,7 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb
if (markDeleteLimiter != null && !markDeleteLimiter.tryAcquire()) {
isDirty = true;
updateLastMarkDeleteEntryToLatest(newMarkDeletePosition, null);
- callback.deleteComplete(ctx);
+ ackCallback.deleteComplete(ctx);
return;
}
@@ -2679,12 +2931,12 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb
internalAsyncMarkDelete(newMarkDeletePosition, null, new MarkDeleteCallback() {
@Override
public void markDeleteComplete(Object ctx) {
- callback.deleteComplete(ctx);
+ ackCallback.deleteComplete(ctx);
}
@Override
public void markDeleteFailed(ManagedLedgerException exception, Object ctx) {
- callback.deleteFailed(exception, ctx);
+ ackCallback.deleteFailed(exception, ctx);
}
}, ctx, null);
@@ -2695,7 +2947,7 @@ public void markDeleteFailed(ManagedLedgerException exception, Object ctx) {
log.debug("[{}] Consumer {} cursor asyncDelete error, counters: consumed {} mdPos {} rdPos {}",
ledger.getName(), name, messagesConsumedCounter, markDeletePosition, readPosition);
}
- callback.deleteFailed(new ManagedLedgerException(e), ctx);
+ ackCallback.deleteFailed(new ManagedLedgerException(e), ctx);
}
}
@@ -2941,6 +3193,11 @@ public void operationFailed(MetaStoreException e) {
private boolean shouldPersistUnackRangesToLedger() {
lock.readLock().lock();
try {
+ // With the per-msgLedger checkpoint log enabled, closing always persists to BK:
+ // holes must never be collapsed into the ZK metadata (the legacy small-state path).
+ if (getConfig().isPersistentUnackedRangesWithPerLedgerEntryEnabled()) {
+ return cursorLedger != null && !isCursorLedgerReadOnly;
+ }
return cursorLedger != null
&& !isCursorLedgerReadOnly
&& getConfig().getMaxUnackedRangesToPersist() > 0
@@ -2968,6 +3225,11 @@ private void persistPositionMetaStore(long cursorsLedgerId, Position position, M
.setMarkDeleteEntryId(position.getEntryId()) //
.setLastActive(lastActive); //
+ if (pendingCheckpointHintLedgerId >= 0) {
+ info.setLastCmCursorLedgerId(pendingCheckpointHintLedgerId);
+ info.setLastCmEntryId(pendingCheckpointHintEntryId);
+ }
+
info.addAllProperties(buildPropertiesMap(properties));
info.addAllCursorProperties(buildStringPropertiesMap(cursorProperties));
if (persistIndividualDeletedMessageRanges) {
@@ -3373,8 +3635,15 @@ private List buildIndividualDeletedMessageRanges() {
AtomicInteger acksSerializedSize = new AtomicInteger(0);
List rangeList = new ArrayList<>();
+ final int maxRanges = getConfig().getMaxUnackedRangesToPersist();
+ final MutableBoolean truncated = new MutableBoolean(false);
individualDeletedMessages.forEachRawRange((lowerKey, lowerValue, upperKey, upperValue) -> {
+ if (rangeList.size() >= maxRanges) {
+ truncated.setTrue();
+ return false;
+ }
+
MLDataFormats.NestedPositionInfo lowerPosition = nestedPositionBuilder
.setLedgerId(lowerKey)
.setEntryId(lowerValue)
@@ -3393,11 +3662,29 @@ private List buildIndividualDeletedMessageRanges() {
acksSerializedSize.addAndGet(messageRange.getSerializedSize());
rangeList.add(messageRange);
- return rangeList.size() <= getConfig().getMaxUnackedRangesToPersist();
+ return true;
});
this.individualDeletedMessagesSerializedSize = acksSerializedSize.get();
individualDeletedMessages.resetDirtyKeys();
+
+ if (truncated.booleanValue()) {
+ ledger.getFactory().getOpenTelemetryManagedCursorStats()
+ .incrementPersistUnackedRangesTruncated(this);
+ if (lastCursorDataFullyPersistable.compareAndSet(true, false)) {
+ int totalRanges = individualDeletedMessages.size();
+ log.warn("[{}]-{} Individually deleted message ranges exceed"
+ + " managedLedgerMaxUnackedRangesToPersist (totalRanges: {}, maxRanges: {},"
+ + " truncated: {}). Acknowledged messages beyond this limit are not persisted"
+ + " and will be replayed on broker restart. Consider raising"
+ + " managedLedgerMaxUnackedRangesToPersist or enabling"
+ + " managedLedgerPersistIndividualAckAsLongArray to reduce the persisted size.",
+ ledger.getName(), name, totalRanges, maxRanges, totalRanges - rangeList.size());
+ }
+ } else {
+ lastCursorDataFullyPersistable.compareAndSet(false, true);
+ }
+
return rangeList;
} finally {
lock.writeLock().unlock();
@@ -3416,7 +3703,8 @@ private List buildBatchEntryDeletio
.BatchedEntryDeletionIndexInfo.newBuilder();
List result = new ArrayList<>();
final var iterator = batchDeletedIndexes.entrySet().iterator();
- while (iterator.hasNext() && result.size() < getConfig().getMaxBatchDeletedIndexToPersist()) {
+ final int maxIndexes = getConfig().getMaxBatchDeletedIndexToPersist();
+ while (iterator.hasNext() && result.size() < maxIndexes) {
final var entry = iterator.next();
nestedPositionBuilder.setLedgerId(entry.getKey().getLedgerId());
nestedPositionBuilder.setEntryId(entry.getKey().getEntryId());
@@ -3430,6 +3718,12 @@ private List buildBatchEntryDeletio
batchDeletedIndexInfoBuilder.addAllDeleteSet(deleteSet);
result.add(batchDeletedIndexInfoBuilder.build());
}
+
+ if (iterator.hasNext()) {
+ ledger.getFactory().getOpenTelemetryManagedCursorStats()
+ .incrementPersistBatchDeletedIndexesTruncated(this);
+ }
+
return result;
} finally {
lock.readLock().unlock();
@@ -3438,6 +3732,72 @@ private List buildBatchEntryDeletio
void persistPositionToLedger(final LedgerHandle lh, MarkDeleteEntry mdEntry, final VoidCallback callback,
boolean ignoreClosedStateAfterFailure) {
+ final long persistStartTimeNanos = System.nanoTime();
+ final VoidCallback persistCallback = new VoidCallback() {
+ @Override
+ public void operationComplete() {
+ recordPersistStats(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - persistStartTimeNanos));
+ callback.operationComplete();
+ }
+
+ @Override
+ public void operationFailed(ManagedLedgerException exception) {
+ recordPersistStats(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - persistStartTimeNanos));
+ callback.operationFailed(exception);
+ }
+ };
+ if (ackPersistence.isPerLedgerEntryPersistEnabled()) {
+ ackPersistence.persist(lh, mdEntry.newPosition, mdEntry.properties, this)
+ .thenAccept(result -> {
+ mdEntry.persistedSuccessfully = true;
+ pendingCheckpointHintLedgerId = lh.getId();
+ pendingCheckpointHintEntryId = result.commitEntryId();
+ mbean.persistToLedger(true);
+ mbean.addWriteCursorLedgerSize(result.totalBytes());
+ rolloverLedgerIfNeeded(lh);
+ persistCallback.operationComplete();
+ })
+ .exceptionally(error -> {
+ Throwable cause = FutureUtil.unwrapCompletionException(error);
+ log.warn("Per-msgLedger checkpoint persist failed, keeping state in memory;"
+ + " retried on the next mark-delete, ledgerId: {}, errorMessage: {}",
+ lh.getId(), cause.getMessage(), cause);
+ mbean.persistToLedger(false);
+ if (cause instanceof BKException) {
+ // BKException fences the handle; next mark-delete creates a fresh ledger.
+ STATE_UPDATER.compareAndSet(ManagedCursorImpl.this, State.Open, State.NoLedger);
+ }
+ // Regular mark-deletes stay tolerated and retried (at-least-once):
+ // ZK must never carry the large ack bitmaps. Cursor resets are different:
+ // their target state has no holes, so an md-only tombstone in ZK is a
+ // complete representation. Fall back to ZK; only if that fails too does
+ // the reset surface as an error (memory untouched, consistent).
+ if (mdEntry.propagatePersistFailure) {
+ persistPositionMetaStore(-1, mdEntry.newPosition, mdEntry.properties,
+ new MetaStoreCallback() {
+ @Override
+ public void operationComplete(Void result, Stat stat) {
+ // The tombstone is durable (in ZK): mark the entry so
+ // the align gate applies the in-memory reset.
+ mdEntry.persistedSuccessfully = true;
+ persistCallback.operationComplete();
+ }
+
+ @Override
+ public void operationFailed(MetaStoreException metaStoreException) {
+ metaStoreException.addSuppressed(cause);
+ persistCallback.operationFailed(createManagedLedgerException(
+ metaStoreException));
+ }
+ }, false);
+ } else {
+ persistCallback.operationComplete();
+ }
+ return null;
+ });
+ return;
+ }
+
Position position = mdEntry.newPosition;
Builder piBuilder = PositionInfo.newBuilder().setLedgerId(position.getLedgerId())
.setEntryId(position.getEntryId())
@@ -3454,10 +3814,27 @@ void persistPositionToLedger(final LedgerHandle lh, MarkDeleteEntry mdEntry, fin
* Do not enable the feature that https://github.com/apache/pulsar/pull/9292 introduced, to avoid serialization
* and deserialization error.
*/
- if (getConfig().isUnackedRangesOpenCacheSetEnabled() && getConfig().isPersistIndividualAckAsLongArray()) {
+
+ if (getConfig().isPersistIndividualAckAsLongArray()) {
lock.readLock().lock();
try {
- internalRanges = individualDeletedMessages.toRanges(getConfig().getMaxUnackedRangesToPersist());
+ MutableBoolean truncatedLedger = new MutableBoolean(false);
+ internalRanges = individualDeletedMessages.toRanges(getConfig().getMaxUnackedRangesToPersist(),
+ skippedLedgers -> {
+ truncatedLedger.setTrue();
+ ledger.getFactory().getOpenTelemetryManagedCursorStats()
+ .incrementPersistUnackedRangesTruncated(this);
+ if (lastCursorDataFullyPersistable.compareAndSet(true, false)) {
+ log.warn("[{}]-{} Truncated individual acks of {} ledgers at persistence:"
+ + " cumulative cardinality exceeds maxUnackedRangesToPersist={}."
+ + " Ack state in skipped ledgers is lost on broker restart.",
+ ledger.getName(), name, skippedLedgers,
+ getConfig().getMaxUnackedRangesToPersist());
+ }
+ });
+ if (!truncatedLedger.get()) {
+ lastCursorDataFullyPersistable.compareAndSet(false, true);
+ }
} catch (Exception e) {
log.warn("[{}]-{} Failed to serialize individualDeletedMessages", ledger.getName(), name, e);
} finally {
@@ -3485,16 +3862,17 @@ void persistPositionToLedger(final LedgerHandle lh, MarkDeleteEntry mdEntry, fin
lh1.getId());
}
+ mdEntry.persistedSuccessfully = true;
rolloverLedgerIfNeeded(lh1);
mbean.persistToLedger(true);
mbean.addWriteCursorLedgerSize(data.length);
- callback.operationComplete();
+ persistCallback.operationComplete();
} else {
if (!ignoreClosedStateAfterFailure && state.isClosed()) {
// After closed the cursor, the in-progress persistence task will get a
// BKException.Code.LedgerClosedException.
- callback.operationFailed(new CursorAlreadyClosedException(String.format("%s %s skipped this"
+ persistCallback.operationFailed(new CursorAlreadyClosedException(String.format("%s %s skipped this"
+ " persistence, because the cursor already closed", ledger.getName(), name)));
return;
}
@@ -3505,7 +3883,7 @@ void persistPositionToLedger(final LedgerHandle lh, MarkDeleteEntry mdEntry, fin
STATE_UPDATER.compareAndSet(ManagedCursorImpl.this, State.Open, State.NoLedger);
// Before giving up, try to persist the position in the metadata store.
- persistPositionToMetaStore(mdEntry, callback);
+ persistPositionToMetaStore(mdEntry, persistCallback);
}
}, null);
}
@@ -3551,6 +3929,7 @@ public void operationComplete(Void result, Stat stat) {
"[{}][{}] Updated cursor in meta store after previous failure in ledger at position"
+ " {}", ledger.getName(), name, newPosition);
}
+ mdEntry.persistedSuccessfully = true;
mbean.persistToZookeeper(true);
callback.operationComplete();
}
@@ -3580,6 +3959,61 @@ boolean shouldCloseLedger(LedgerHandle lh) {
}
}
+ /**
+ * GC for old cursor ledgers. Uses the simplified {@code maxHold} model — when
+ * mark-delete has advanced past the highest msgLedgerId that still holds individual or batch
+ * acks,
+ * all referenced AckStateRefs are stale and every old cursor ledger can be deleted.
+ *
+ * This is an all-or-nothing GC: it either deletes all tracked old ledgers or none.
+ * Unreferenced ledgers that were never tracked (e.g. from before a restart) leak —
+ * acceptable because they hold no live data.
+ */
+ @VisibleForTesting
+ void gcOldCursorLedgers() {
+ if (!ackPersistence.isPerLedgerEntryPersistEnabled() || allCursorLedgerIds.isEmpty()) {
+ return;
+ }
+ long mdLedgerId = markDeletePosition.getLedgerId();
+ // Find the highest msgLedgerId that currently has individual acks above mark-delete.
+ // individualDeletedMessages.rangeBitmapMap is not thread-safe, so the iteration must
+ // hold the cursor readLock to prevent concurrent mark-delete / delete modifications.
+ long[] maxHold = {-1};
+ lock.readLock().lock();
+ try {
+ individualDeletedMessages.forEachActiveLedger(id -> {
+ if (id > maxHold[0]) {
+ maxHold[0] = id;
+ }
+ });
+ if (batchDeletedIndexes != null) {
+ batchDeletedIndexes.keySet().forEach(pos -> {
+ long id = pos.getLedgerId();
+ if (id > maxHold[0]) {
+ maxHold[0] = id;
+ }
+ });
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ if (maxHold[0] >= 0 && mdLedgerId < maxHold[0]) {
+ return;
+ }
+ for (long id : new ArrayList<>(allCursorLedgerIds)) {
+ log.debug("GC old cursor ledger - maxHold cleared, ledgerId: {}", id);
+ bookkeeper.asyncDeleteLedger(id, (rc, ctx) -> {
+ if (rc == BKException.Code.OK || rc == BKException.Code.NoSuchLedgerExistsException) {
+ allCursorLedgerIds.remove(id);
+ } else {
+ log.warn("Failed to GC old cursor ledger, will retry on next rollover, ledgerId: {}, "
+ + "errorMessage: {}",
+ id, BKException.getMessage(rc));
+ }
+ }, null);
+ }
+ }
+
void switchToNewLedger(final LedgerHandle lh, final VoidCallback callback) {
if (log.isDebugEnabled()) {
log.debug("[{}] Switching cursor {} to ledger {}", ledger.getName(), name, lh.getId());
@@ -3597,7 +4031,12 @@ public void operationComplete(Void result, Stat stat) {
// At this point the position had already been safely markdeleted
callback.operationComplete();
- asyncDeleteLedger(oldLedger);
+ if (ackPersistence.isPerLedgerEntryPersistEnabled() && oldLedger != null) {
+ allCursorLedgerIds.add(oldLedger.getId());
+ gcOldCursorLedgers();
+ } else {
+ asyncDeleteLedger(oldLedger);
+ }
}
@Override
@@ -3982,6 +4421,12 @@ public ManagedLedger getManagedLedger() {
return this.ledger;
}
+ @Override
+ public void scheduleReadCallback(Runnable callback, long delay, TimeUnit unit) {
+ ledger.getScheduledExecutor().schedule(
+ catchingAndLoggingThrowables(() -> ledger.getExecutor().execute(callback)), delay, unit);
+ }
+
@Override
public Range getLastIndividualDeletedRange() {
lock.readLock().lock();
@@ -4146,7 +4591,8 @@ public ManagedLedgerInternalStats.CursorStats getCursorStats() {
cs.messagesConsumedCounter = getMessagesConsumedCounter();
cs.cursorLedger = getCursorLedger();
cs.cursorLedgerLastEntry = getCursorLedgerLastEntry();
- cs.individuallyDeletedMessages = getIndividuallyDeletedMessages();
+ cs.individualDeletedMessagesCount = getNumberOfIndividualDeletedMessages();
+ cs.firstIndividualDeletedMessage = getFirstIndividualDeletedMessage();
cs.lastLedgerSwitchTimestamp = DateFormatter.format(getLastLedgerSwitchTimestamp());
cs.state = getState();
cs.active = isActive();
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorMXBeanImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorMXBeanImpl.java
index a183c0d61ce16..69b374870aeeb 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorMXBeanImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorMXBeanImpl.java
@@ -34,6 +34,17 @@ public class ManagedCursorMXBeanImpl implements ManagedCursorMXBean {
private final LongAdder writeCursorLedgerLogicalSize = new LongAdder();
private final LongAdder readCursorLedgerSize = new LongAdder();
+ private final LongAdder ackCount = new LongAdder();
+ private final LongAdder ackLatencyTotalMillis = new LongAdder();
+
+ private final LongAdder persistCount = new LongAdder();
+ private final LongAdder persistLatencyTotalMillis = new LongAdder();
+
+ private final LongAdder recoverCount = new LongAdder();
+ private final LongAdder recoverSucceed = new LongAdder();
+ private final LongAdder recoverErrors = new LongAdder();
+ private final LongAdder recoverLatencyTotalMillis = new LongAdder();
+
private final ManagedCursor managedCursor;
public ManagedCursorMXBeanImpl(ManagedCursor managedCursor) {
@@ -114,4 +125,70 @@ public long getWriteCursorLedgerLogicalSize() {
public long getReadCursorLedgerSize() {
return readCursorLedgerSize.longValue();
}
+
+ @Override
+ public void recordAck(long latencyMillis) {
+ ackCount.increment();
+ ackLatencyTotalMillis.add(latencyMillis);
+ }
+
+ @Override
+ public long getAckCount() {
+ return ackCount.longValue();
+ }
+
+ @Override
+ public double getAckLatencyAvgMillis() {
+ long count = ackCount.longValue();
+ return count == 0 ? 0 : ackLatencyTotalMillis.longValue() / (double) count;
+ }
+
+ @Override
+ public void recordPersist(long latencyMillis) {
+ persistCount.increment();
+ persistLatencyTotalMillis.add(latencyMillis);
+ }
+
+ @Override
+ public long getPersistCount() {
+ return persistCount.longValue();
+ }
+
+ @Override
+ public double getPersistLatencyAvgMillis() {
+ long count = persistCount.longValue();
+ return count == 0 ? 0 : persistLatencyTotalMillis.longValue() / (double) count;
+ }
+
+ @Override
+ public void recordRecover(long latencyMillis, boolean success) {
+ recoverCount.increment();
+ recoverLatencyTotalMillis.add(latencyMillis);
+ if (success) {
+ recoverSucceed.increment();
+ } else {
+ recoverErrors.increment();
+ }
+ }
+
+ @Override
+ public long getRecoverCount() {
+ return recoverCount.longValue();
+ }
+
+ @Override
+ public long getRecoverSucceed() {
+ return recoverSucceed.longValue();
+ }
+
+ @Override
+ public long getRecoverErrors() {
+ return recoverErrors.longValue();
+ }
+
+ @Override
+ public double getRecoverLatencyAvgMillis() {
+ long count = recoverCount.longValue();
+ return count == 0 ? 0 : recoverLatencyTotalMillis.longValue() / (double) count;
+ }
}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
index bcca3fb9a5286..8690653ec2477 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
@@ -19,7 +19,7 @@
package org.apache.bookkeeper.mledger.impl;
import static org.apache.bookkeeper.mledger.ManagedLedgerException.getManagedLedgerException;
-import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.NULL_OFFLOAD_PROMISE;
+import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.AUTOMATIC_OFFLOAD_TRIGGER;
import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables;
import com.google.common.base.Predicates;
import com.google.common.collect.BoundType;
@@ -250,6 +250,10 @@ private ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore,
openTelemetryManagedCursorStats = new OpenTelemetryManagedCursorStats(openTelemetry, this);
}
+ public OpenTelemetryManagedCursorStats getOpenTelemetryManagedCursorStats() {
+ return openTelemetryManagedCursorStats;
+ }
+
static class DefaultBkFactory implements BookkeeperFactoryForCustomEnsemblePlacementPolicy, AutoCloseable {
private final BookKeeper bkClient;
@@ -430,7 +434,7 @@ public void initializeComplete() {
future.complete(newledger);
// May need to trigger offloading
if (config.isTriggerOffloadOnTopicLoad()) {
- newledger.maybeOffloadInBackground(NULL_OFFLOAD_PROMISE);
+ newledger.maybeOffloadInBackground(AUTOMATIC_OFFLOAD_TRIGGER);
}
});
}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
index 7c86f4396e39f..8697a7ba8ab7e 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
@@ -228,8 +228,20 @@ public class ManagedLedgerImpl implements ManagedLedger, CreateCallback {
protected final CallbackMutex trimmerMutex = new CallbackMutex();
protected final CallbackMutex offloadMutex = new CallbackMutex();
- public static final CompletableFuture NULL_OFFLOAD_PROMISE = CompletableFuture
+ private final AutomaticOffloadTriggerController automaticOffloadTriggerController =
+ new AutomaticOffloadTriggerController();
+ // Identity sentinel for automatic offload requests. The completed Position value is not used.
+ public static final CompletableFuture AUTOMATIC_OFFLOAD_TRIGGER = CompletableFuture
.completedFuture(PositionFactory.LATEST);
+
+ private enum OffloadRequestSource {
+ AUTOMATIC,
+ EXPLICIT
+ }
+
+ private record OffloadThresholds(long thresholdInBytes, long thresholdInSeconds) {
+ }
+
@VisibleForTesting
@Getter
protected volatile LedgerHandle currentLedger;
@@ -1955,7 +1967,7 @@ synchronized void ledgerClosed(final LedgerHandle lh, Long lastAddConfirmed, Led
trimConsumedLedgersInBackground();
- maybeOffloadInBackground(NULL_OFFLOAD_PROMISE);
+ maybeOffloadInBackground(AUTOMATIC_OFFLOAD_TRIGGER);
createLedgerAfterClosed(ledgerRollReason);
}
@@ -2443,10 +2455,13 @@ private void internalReadFromLedger(ReadHandle ledger, OpReadEntry opReadEntry)
}
}
- // If all messages in [firstEntry...lastEntry] are filter out,
- // then manual call internalReadEntriesComplete to advance read position.
+ // If all positions in [firstEntry...lastEntry] are filtered out, advance the read position
+ // without issuing a ledger read.
if (firstValidEntry == -1L) {
- final var nextReadPosition = PositionFactory.create(ledger.getId(), lastEntry).getNext();
+ final Position lastScannedPosition = PositionFactory.create(ledger.getId(), lastEntry);
+ // The whole scan window was skipped. If the last scanned position is individually deleted,
+ // hop over its deleted range instead of advancing one window at a time.
+ final Position nextReadPosition = opReadEntry.cursor.getNextAvailablePosition(lastScannedPosition);
opReadEntry.updateReadPosition(nextReadPosition);
opReadEntry.checkReadCompletion();
return;
@@ -2846,22 +2861,73 @@ private void scheduleDeferredTrimming(boolean isTruncate, CompletableFuture p
}
public void maybeOffloadInBackground(CompletableFuture promise) {
- if (getOffloadPoliciesIfAppendable().isEmpty()) {
+ if (promise == AUTOMATIC_OFFLOAD_TRIGGER) {
+ if (automaticOffloadTriggerController.requestRun()) {
+ startAutomaticOffload();
+ }
+ return;
+ }
+
+ maybeOffloadInBackground(promise, OffloadRequestSource.EXPLICIT);
+ }
+
+ private void startAutomaticOffload() {
+ CompletableFuture automaticOffloadCompletion = new CompletableFuture<>();
+ automaticOffloadCompletion.whenComplete((res, ex) -> finishAutomaticOffload(ex));
+ try {
+ maybeOffloadInBackground(automaticOffloadCompletion, OffloadRequestSource.AUTOMATIC);
+ } catch (RuntimeException e) {
+ automaticOffloadCompletion.completeExceptionally(e);
+ }
+ }
+
+ private void maybeOffloadInBackground(CompletableFuture promise, OffloadRequestSource source) {
+ Optional offloadThresholds = getOffloadThresholds();
+ if (offloadThresholds.isEmpty()) {
+ if (source == OffloadRequestSource.AUTOMATIC) {
+ promise.complete(PositionFactory.LATEST);
+ }
return;
}
- final OffloadPolicies policies = config.getLedgerOffloader().getOffloadPolicies();
+ OffloadThresholds thresholds = offloadThresholds.get();
+ try {
+ executor.execute(() -> maybeOffload(thresholds.thresholdInBytes(), thresholds.thresholdInSeconds(),
+ promise, source));
+ } catch (RuntimeException e) {
+ promise.completeExceptionally(e);
+ }
+ }
+
+ private Optional getOffloadThresholds() {
+ Optional optionalOffloadPolicies = getOffloadPoliciesIfAppendable();
+ if (optionalOffloadPolicies.isEmpty()) {
+ return Optional.empty();
+ }
+
+ final OffloadPolicies policies = optionalOffloadPolicies.get();
final long offloadThresholdInBytes =
Optional.ofNullable(policies.getManagedLedgerOffloadThresholdInBytes()).orElse(-1L);
final long offloadThresholdInSeconds =
Optional.ofNullable(policies.getManagedLedgerOffloadThresholdInSeconds()).orElse(-1L);
if (offloadThresholdInBytes >= 0 || offloadThresholdInSeconds >= 0) {
- executor.execute(() -> maybeOffload(offloadThresholdInBytes, offloadThresholdInSeconds, promise));
+ return Optional.of(new OffloadThresholds(offloadThresholdInBytes, offloadThresholdInSeconds));
+ }
+
+ return Optional.empty();
+ }
+
+ private void finishAutomaticOffload(Throwable exception) {
+ if (exception != null && log.isDebugEnabled()) {
+ log.debug("Failed to automatically offload ledgers", exception);
+ }
+ if (automaticOffloadTriggerController.completeRun()) {
+ startAutomaticOffload();
}
}
private void maybeOffload(long offloadThresholdInBytes, long offloadThresholdInSeconds,
- CompletableFuture finalPromise) {
+ CompletableFuture finalPromise, OffloadRequestSource source) {
if (getOffloadPoliciesIfAppendable().isEmpty()) {
String msg = String.format("[%s] Nothing to offload due to offloader or offloadPolicies is NULL", name);
finalPromise.completeExceptionally(new IllegalArgumentException(msg));
@@ -2876,8 +2942,12 @@ private void maybeOffload(long offloadThresholdInBytes, long offloadThresholdInS
}
if (!offloadMutex.tryLock()) {
- scheduledExecutor.schedule(() -> maybeOffloadInBackground(finalPromise),
- 100, TimeUnit.MILLISECONDS);
+ try {
+ scheduledExecutor.schedule(() -> maybeOffloadInBackground(finalPromise, source),
+ 100, TimeUnit.MILLISECONDS);
+ } catch (RuntimeException e) {
+ finalPromise.completeExceptionally(e);
+ }
return;
}
@@ -2967,12 +3037,11 @@ void internalTrimConsumedLedgers(CompletableFuture promise) {
private Optional getOffloadPoliciesIfAppendable() {
LedgerOffloader ledgerOffloader = config.getLedgerOffloader();
- if (ledgerOffloader == null
- || !ledgerOffloader.isAppendable()
- || ledgerOffloader.getOffloadPolicies() == null) {
+ if (ledgerOffloader == null || !ledgerOffloader.isAppendable()) {
return Optional.empty();
}
- return Optional.ofNullable(ledgerOffloader.getOffloadPolicies());
+ OffloadPolicies offloadPolicies = ledgerOffloader.getOffloadPolicies();
+ return Optional.ofNullable(offloadPolicies);
}
@VisibleForTesting
@@ -4941,7 +5010,9 @@ public CompletableFuture getManagedLedgerInternalSta
stats.lastConfirmedEntry = this.getLastConfirmedEntry().toString();
stats.state = this.getState().toString();
- stats.cursors = new HashMap();
+ stats.properties = new HashMap<>(propertiesMap);
+
+ stats.cursors = new HashMap<>();
this.getCursors().forEach(c -> {
ManagedCursorImpl cursor = (ManagedCursorImpl) c;
PersistentTopicInternalStats.CursorStats cs = new PersistentTopicInternalStats.CursorStats();
@@ -4952,7 +5023,8 @@ public CompletableFuture getManagedLedgerInternalSta
cs.messagesConsumedCounter = cursor.getMessagesConsumedCounter();
cs.cursorLedger = cursor.getCursorLedger();
cs.cursorLedgerLastEntry = cursor.getCursorLedgerLastEntry();
- cs.individuallyDeletedMessages = cursor.getIndividuallyDeletedMessages();
+ cs.individualDeletedMessagesCount = cursor.getNumberOfIndividualDeletedMessages();
+ cs.firstIndividualDeletedMessage = cursor.getFirstIndividualDeletedMessage();
cs.lastLedgerSwitchTimestamp = DateFormatter.format(cursor.getLastLedgerSwitchTimestamp());
cs.state = cursor.getState();
cs.active = cursor.isActive();
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
index 611d9d60202cd..7b122691fe46f 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
@@ -126,6 +126,17 @@ public void getManagedLedgerInfo(String ledgerName, boolean createIfMissing, Map
}
callback.operationComplete(ledgerBuilder.build(), stat);
}).exceptionally(ex -> {
+ if (FutureUtil.unwrapCompletionException(ex)
+ instanceof MetadataStoreException.BadVersionException) {
+ // The z-node was created concurrently after the read above returned
+ // "not found". This happens for example when the broker creates the
+ // partitions of a topic while the same topic is being loaded. Read the
+ // z-node back instead of failing the managed ledger initialization.
+ log.info("Managed ledger path '{}' was concurrently created, reading it "
+ + "back", path);
+ getManagedLedgerInfo(ledgerName, false, null, callback);
+ return null;
+ }
callback.operationFailed(getException(ex));
return null;
});
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpenTelemetryManagedCursorStats.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpenTelemetryManagedCursorStats.java
index ec73c9d5e5eb2..ffaed66ca20dd 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpenTelemetryManagedCursorStats.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpenTelemetryManagedCursorStats.java
@@ -20,7 +20,10 @@
import com.google.common.collect.Streams;
import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.api.common.Attributes;
import io.opentelemetry.api.metrics.BatchCallback;
+import io.opentelemetry.api.metrics.DoubleHistogram;
+import io.opentelemetry.api.metrics.LongCounter;
import io.opentelemetry.api.metrics.ObservableLongMeasurement;
import org.apache.bookkeeper.mledger.ManagedCursor;
import org.apache.bookkeeper.mledger.ManagedLedger;
@@ -55,6 +58,40 @@ public class OpenTelemetryManagedCursorStats implements AutoCloseable {
public static final String INCOMING_BYTE_COUNTER = "pulsar.broker.managed_ledger.cursor.incoming.size";
private final ObservableLongMeasurement incomingByteCounter;
+ // Broker-level counters incremented when cursor persistence silently truncates ack state.
+ // See managedLedgerMaxUnackedRangesToPersist and managedLedgerMaxBatchDeletedIndexToPersist.
+ public static final String PERSIST_UNACKED_RANGES_TRUNCATED =
+ "pulsar.broker.managed_ledger.cursor.persist.unacked_ranges.truncated";
+ private final LongCounter persistUnackedRangesTruncated;
+
+ public static final String PERSIST_BATCH_DELETED_INDEXES_TRUNCATED =
+ "pulsar.broker.managed_ledger.cursor.persist.batch_deleted_indexes.truncated";
+ private final LongCounter persistBatchDeletedIndexesTruncated;
+
+ // Replaces ['brk_ml_cursor_ackCount']
+ public static final String ACK_OPERATION_COUNTER =
+ "pulsar.broker.managed_ledger.cursor.ack.operation.count";
+ private final ObservableLongMeasurement ackOperationCounter;
+
+ // Replaces ['brk_ml_cursor_ackLatencyAvgMs']
+ public static final String ACK_LATENCY = "pulsar.broker.managed_ledger.cursor.ack.latency";
+ private final DoubleHistogram ackLatency;
+
+ // Replaces ['brk_ml_cursor_persistLatencyAvgMs']; the count is already covered by
+ // PERSIST_OPERATION_COUNTER.
+ public static final String PERSIST_LATENCY = "pulsar.broker.managed_ledger.cursor.persist.latency";
+ private final DoubleHistogram persistLatency;
+
+ // Replaces ['brk_ml_cursor_recoverCount', 'brk_ml_cursor_recoverSucceed',
+ // 'brk_ml_cursor_recoverErrors'].
+ public static final String RECOVER_OPERATION_COUNTER =
+ "pulsar.broker.managed_ledger.cursor.recover.operation.count";
+ private final ObservableLongMeasurement recoverOperationCounter;
+
+ // Replaces ['brk_ml_cursor_recoverLatencyAvgMs']
+ public static final String RECOVER_LATENCY = "pulsar.broker.managed_ledger.cursor.recover.latency";
+ private final DoubleHistogram recoverLatency;
+
private final BatchCallback batchCallback;
public OpenTelemetryManagedCursorStats(OpenTelemetry openTelemetry, ManagedLedgerFactoryImpl factory) {
@@ -96,6 +133,49 @@ public OpenTelemetryManagedCursorStats(OpenTelemetry openTelemetry, ManagedLedge
.setDescription("The total amount of data read from the ledger.")
.buildObserver();
+ persistUnackedRangesTruncated = meter
+ .counterBuilder(PERSIST_UNACKED_RANGES_TRUNCATED)
+ .setUnit("{truncation}")
+ .setDescription("The number of times a cursor exceeded"
+ + " managedLedgerMaxUnackedRangesToPersist, causing ack state to be truncated"
+ + " at persistence. Ack state beyond the limit is lost on broker restart.")
+ .build();
+
+ persistBatchDeletedIndexesTruncated = meter
+ .counterBuilder(PERSIST_BATCH_DELETED_INDEXES_TRUNCATED)
+ .setUnit("{truncation}")
+ .setDescription("The number of times a cursor exceeded"
+ + " managedLedgerMaxBatchDeletedIndexToPersist, causing batch deleted index state"
+ + " to be truncated at persistence. State beyond the limit is lost on broker restart.")
+ .build();
+
+ ackOperationCounter = meter
+ .counterBuilder(ACK_OPERATION_COUNTER)
+ .setUnit("{operation}")
+ .setDescription("The number of cursor acknowledgment operations.")
+ .buildObserver();
+
+ ackLatency = meter.histogramBuilder(ACK_LATENCY)
+ .setUnit("s")
+ .setDescription("Cursor acknowledgment operation latency.")
+ .build();
+
+ persistLatency = meter.histogramBuilder(PERSIST_LATENCY)
+ .setUnit("s")
+ .setDescription("Cursor persist latency: the checkpoint write to the cursor ledger.")
+ .build();
+
+ recoverOperationCounter = meter
+ .counterBuilder(RECOVER_OPERATION_COUNTER)
+ .setUnit("{operation}")
+ .setDescription("The number of cursor recovery operations.")
+ .buildObserver();
+
+ recoverLatency = meter.histogramBuilder(RECOVER_LATENCY)
+ .setUnit("s")
+ .setDescription("Cursor recovery latency.")
+ .build();
+
batchCallback = meter.batchCallback(() -> factory.getManagedLedgers()
.values()
.stream()
@@ -107,7 +187,9 @@ public OpenTelemetryManagedCursorStats(OpenTelemetry openTelemetry, ManagedLedge
nonContiguousMessageRangeCounter,
outgoingByteCounter,
outgoingByteLogicalCounter,
- incomingByteCounter);
+ incomingByteCounter,
+ ackOperationCounter,
+ recoverOperationCounter);
}
@Override
@@ -115,6 +197,14 @@ public void close() {
batchCallback.close();
}
+ public void incrementPersistUnackedRangesTruncated(ManagedCursor cursor) {
+ persistUnackedRangesTruncated.add(1, cursor.getManagedCursorAttributes().getAttributes());
+ }
+
+ public void incrementPersistBatchDeletedIndexesTruncated(ManagedCursor cursor) {
+ persistBatchDeletedIndexesTruncated.add(1, cursor.getManagedCursorAttributes().getAttributes());
+ }
+
private void recordMetrics(ManagedCursor cursor) {
var stats = cursor.getStats();
var cursorAttributesSet = cursor.getManagedCursorAttributes();
@@ -133,5 +223,40 @@ private void recordMetrics(ManagedCursor cursor) {
outgoingByteCounter.record(stats.getWriteCursorLedgerSize(), attributes);
outgoingByteLogicalCounter.record(stats.getWriteCursorLedgerLogicalSize(), attributes);
incomingByteCounter.record(stats.getReadCursorLedgerSize(), attributes);
+
+ ackOperationCounter.record(stats.getAckCount(), attributes);
+ recoverOperationCounter.record(stats.getRecoverSucceed(), attributesSucceed);
+ recoverOperationCounter.record(stats.getRecoverErrors(), attributesFailed);
+ }
+
+ public void recordAckLatency(ManagedCursor cursor, double seconds) {
+ var attributes = attributesOrNull(cursor);
+ if (attributes != null) {
+ ackLatency.record(seconds, attributes);
+ }
+ }
+
+ public void recordPersistLatency(ManagedCursor cursor, double seconds) {
+ var attributes = attributesOrNull(cursor);
+ if (attributes != null) {
+ persistLatency.record(seconds, attributes);
+ }
+ }
+
+ public void recordRecoverLatency(ManagedCursor cursor, double seconds) {
+ var attributes = attributesOrNull(cursor);
+ if (attributes != null) {
+ recoverLatency.record(seconds, attributes);
+ }
+ }
+
+ private static Attributes attributesOrNull(ManagedCursor cursor) {
+ try {
+ return cursor.getManagedCursorAttributes().getAttributes();
+ } catch (Exception e) {
+ // Attribute resolution throws for non-topic-named managed ledgers (e.g. system
+ // ledgers); recording telemetry must never break the caller's callback chain.
+ return null;
+ }
}
}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionRangeSet.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionRangeSet.java
new file mode 100644
index 0000000000000..82fcbf14264a4
--- /dev/null
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionRangeSet.java
@@ -0,0 +1,599 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger.impl;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.BoundType;
+import com.google.common.collect.Range;
+import io.netty.buffer.Unpooled;
+import it.unimi.dsi.fastutil.longs.Long2ObjectRBTreeMap;
+import it.unimi.dsi.fastutil.longs.Long2ObjectSortedMap;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.LongConsumer;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.PositionFactory;
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.pulsar.common.util.collections.LongBitmap;
+import org.apache.pulsar.common.util.collections.LongBitmaps;
+import org.apache.pulsar.common.util.collections.LongPairRangeSet;
+
+/**
+ * Tracks deleted-message positions as ranges of {@link Position}s.
+ *
+ * The implementation stores positions in a two-level structure:
+ * the ledger id is used as the map key, and the corresponding entry ids are stored in a
+ * {@link LongBitmap}. Bit {@code n} in the bitmap of ledger {@code L} represents the position
+ * {@code (L, n)}.
+ *
+ *
Thread Safety
+ *
+ * This class is not thread-safe. All methods require the caller to provide external
+ * synchronization. In normal usage, callers must hold the owning {@link ManagedCursorImpl}'s
+ * cursor lock before accessing this class.
+ *
+ *
The class intentionally does not maintain internal locking. The cursor lock is the single
+ * synchronization boundary for both this structure and related cursor state.
+ *
+ *
Persistence Compatibility
+ *
+ * The persisted representation remains compatible with the existing format using
+ * {@link LongBitmap#serializeToLongArray()} and {@link LongBitmap#deserializeFromLongArray(long[])},
+ * which are compatible with the BitSet long[] format.
+ *
+ *
Entry ids are stored as bitmap indexes and therefore use {@code int} values. This is safe
+ * because {@code managedLedgerMaxEntriesPerLedger} is an {@code int}, so valid entry ids are within
+ * {@code [0, Integer.MAX_VALUE]}.
+ */
+@Slf4j
+class PositionRangeSet implements LongPairRangeSet {
+
+ private static final long EARLIEST_LEDGER_ID = -1L;
+ private static final long EARLIEST_ENTRY_ID = -1L;
+ private static final long LATEST_LEDGER_ID = Long.MAX_VALUE;
+ private static final long LATEST_ENTRY_ID = Long.MAX_VALUE;
+
+ /**
+ * Maps ledger ID to a bitmap of deleted entry IDs within that ledger.
+ * Bit {@code n} in the bitmap represents entry {@code n} in the ledger.
+ */
+ private final Long2ObjectSortedMap rangeBitmapMap = new Long2ObjectRBTreeMap<>();
+ private final LongPairConsumer consumer;
+ private final boolean enableMultiEntry;
+
+ private final LongBitmap dirtyLedgers = LongBitmaps.create();
+
+ private int cachedSize = 0;
+ private String cachedToString = "[]";
+ private boolean updatedAfterCachedForSize = true;
+ private boolean updatedAfterCachedForToString = true;
+ private long totalCardinality = 0;
+
+ PositionRangeSet(LongPairConsumer consumer, boolean enableMultiEntry) {
+ this.consumer = consumer;
+ this.enableMultiEntry = enableMultiEntry;
+ }
+
+ private static long lastPresentValue(LongBitmap bitmap) {
+ return bitmap.lastPresentValue();
+ }
+
+ @Override
+ public void addOpenClosed(long lowerLedgerId, long lowerEntryIdOpen, long upperLedgerId, long upperEntryId) {
+ if (enableMultiEntry) {
+ markDirty(lowerLedgerId, upperLedgerId);
+ }
+ long lowerEntryId = lowerEntryIdOpen + 1;
+ if (lowerLedgerId != upperLedgerId) {
+ // Extend lower ledger's bitmap only if it already exists and has bits at/after lowerEntryId;
+ // otherwise we'd invent acknowledgements that never happened (e.g. (2:10..4:10] must not
+ // touch 2:10 if ledger 2 was empty).
+ if (isValid(lowerLedgerId, lowerEntryId)) {
+ LongBitmap rangeBitmap = rangeBitmapMap.get(lowerLedgerId);
+ if (rangeBitmap != null) {
+ long lastEntryId = rangeBitmap.lastPresentValue();
+ if (lastEntryId > lowerEntryIdOpen) {
+ addRange(rangeBitmap, lowerEntryId, Math.max(lastEntryId, lowerEntryId) + 1);
+ }
+ }
+ }
+ if (isValid(upperLedgerId, upperEntryId)) {
+ LongBitmap rangeBitmap = rangeBitmapMap.computeIfAbsent(upperLedgerId, k -> LongBitmaps.create());
+ addRange(rangeBitmap, 0, upperEntryId + 1);
+ }
+ } else {
+ LongBitmap rangeBitmap = rangeBitmapMap.computeIfAbsent(lowerLedgerId, k -> LongBitmaps.create());
+ addRange(rangeBitmap, lowerEntryId, upperEntryId + 1);
+ }
+ invalidateCaches();
+ }
+
+ private void addRange(LongBitmap bitmap, long from, long to) {
+ long before = bitmap.cardinality();
+ bitmap.add(from, to);
+ totalCardinality += bitmap.cardinality() - before;
+ }
+
+ private void removeRange(LongBitmap bitmap, long from, long to) {
+ long before = bitmap.cardinality();
+ bitmap.remove(from, to);
+ totalCardinality -= before - bitmap.cardinality();
+ }
+
+ private void clearSubMapCardinality(Long2ObjectSortedMap subMap) {
+ for (LongBitmap bitmap : subMap.values()) {
+ totalCardinality -= bitmap.cardinality();
+ }
+ subMap.clear();
+ }
+
+ @Override
+ public boolean contains(long ledgerId, long entryId) {
+ LongBitmap rangeBitmap = rangeBitmapMap.get(ledgerId);
+ if (rangeBitmap != null) {
+ return rangeBitmap.contains(getSafeEntry(entryId));
+ }
+ return false;
+ }
+
+ @Override
+ public Range rangeContaining(long ledgerId, long entryId) {
+ LongBitmap rangeBitmap = rangeBitmapMap.get(ledgerId);
+ if (rangeBitmap == null || !rangeBitmap.contains(getSafeEntry(entryId))) {
+ return null;
+ }
+ long safeEntryId = getSafeEntry(entryId);
+ long lowerEntryId = rangeBitmap.previousAbsentValue(safeEntryId) + 1;
+ Position lower = consumer.apply(ledgerId, lowerEntryId);
+ long nextAbsentEntryId = rangeBitmap.nextAbsentValue(safeEntryId);
+ Position upper = consumer.apply(ledgerId, Math.max(nextAbsentEntryId - 1, lowerEntryId));
+ return Range.closed(lower, upper);
+ }
+
+ @Override
+ public void removeAtMost(long ledgerId, long entryId) {
+ if (enableMultiEntry && ledgerId >= 0) {
+ long end = Math.min(ledgerId + 1L, (long) Integer.MAX_VALUE + 1);
+ dirtyLedgers.remove(0, end);
+ }
+ remove(Range.atMost(PositionFactory.create(ledgerId, entryId)));
+ }
+
+ @Override
+ public boolean isEmpty() {
+ if (rangeBitmapMap.isEmpty()) {
+ return true;
+ }
+ for (LongBitmap bitmap : rangeBitmapMap.values()) {
+ if (!bitmap.isEmpty()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public void clear() {
+ rangeBitmapMap.clear();
+ totalCardinality = 0;
+ resetDirtyKeys();
+ invalidateCaches();
+ }
+
+ @Override
+ public Range span() {
+ if (rangeBitmapMap.isEmpty()) {
+ return null;
+ }
+ long firstLedgerId = rangeBitmapMap.firstLongKey();
+ long lastLedgerId = rangeBitmapMap.lastLongKey();
+ LongBitmap firstBitmap = rangeBitmapMap.get(firstLedgerId);
+ LongBitmap lastBitmap = rangeBitmapMap.get(lastLedgerId);
+ long firstEntryId = firstBitmap.nextPresentValue(0);
+ long lastEntryId = lastBitmap.lastPresentValue();
+ return Range.openClosed(consumer.apply(firstLedgerId, firstEntryId - 1),
+ consumer.apply(lastLedgerId, lastEntryId));
+ }
+
+ @Override
+ public List> asRanges() {
+ List> ranges = new ArrayList<>();
+ forEach(range -> {
+ ranges.add(range);
+ return true;
+ });
+ return ranges;
+ }
+
+ @Override
+ public void forEach(RangeProcessor action) {
+ forEach(action, consumer);
+ }
+
+ @Override
+ public void forEach(RangeProcessor action, LongPairConsumer consumerParam) {
+ forEachRawRange((lowerLedgerId, lowerEntryId, upperLedgerId, upperEntryId) -> {
+ Range range = Range.openClosed(
+ consumerParam.apply(lowerLedgerId, lowerEntryId),
+ consumerParam.apply(upperLedgerId, upperEntryId));
+ return action.process(range);
+ });
+ }
+
+ @Override
+ public void forEachRawRange(RawRangeProcessor processor) {
+ AtomicBoolean completed = new AtomicBoolean(false);
+ rangeBitmapMap.forEach((ledgerId, bitmap) -> {
+ if (completed.get() || bitmap.isEmpty()) {
+ return;
+ }
+ long firstEntryId = bitmap.nextPresentValue(0);
+ long lastEntryId = bitmap.lastPresentValue();
+ long currentEntryId = firstEntryId;
+ while (currentEntryId != -1 && currentEntryId <= lastEntryId) {
+ long nextAbsentEntryId = bitmap.nextAbsentValue(currentEntryId);
+ if (!processor.processRawRange(ledgerId, currentEntryId - 1, ledgerId, nextAbsentEntryId - 1)) {
+ completed.set(true);
+ break;
+ }
+ if (nextAbsentEntryId > Integer.MAX_VALUE) {
+ break;
+ }
+ currentEntryId = bitmap.nextPresentValue(nextAbsentEntryId);
+ }
+ });
+ }
+
+ @Override
+ public Range firstRange() {
+ if (rangeBitmapMap.isEmpty()) {
+ return null;
+ }
+ long firstLedgerId = rangeBitmapMap.firstLongKey();
+ LongBitmap firstBitmap = rangeBitmapMap.get(firstLedgerId);
+ long lowerEntryId = firstBitmap.nextPresentValue(0);
+ long upperEntryId = Math.max(lowerEntryId, firstBitmap.nextAbsentValue(lowerEntryId) - 1);
+ return Range.openClosed(consumer.apply(firstLedgerId, lowerEntryId - 1),
+ consumer.apply(firstLedgerId, upperEntryId));
+ }
+
+ @Override
+ public Range lastRange() {
+ if (rangeBitmapMap.isEmpty()) {
+ return null;
+ }
+ long lastLedgerId = rangeBitmapMap.lastLongKey();
+ LongBitmap lastBitmap = rangeBitmapMap.get(lastLedgerId);
+ long upperEntryId = lastBitmap.lastPresentValue();
+ long lowerEntryId = Math.min(lastBitmap.previousAbsentValue(upperEntryId), upperEntryId);
+ return Range.openClosed(consumer.apply(lastLedgerId, lowerEntryId),
+ consumer.apply(lastLedgerId, upperEntryId));
+ }
+
+ @Override
+ public Map toRanges(int maxRanges) {
+ return toRanges(maxRanges, null);
+ }
+
+ /**
+ * Serializes the individual-deleted positions as per-ledger bitmaps, including ledgers only while
+ * the cumulative cardinality stays within {@code maxRanges}. Ledgers beyond the limit are skipped
+ * silently (their ack state will be lost on broker restart); when {@code truncationNotifier} is
+ * provided it is invoked once with the number of skipped ledgers so callers can record the loss.
+ */
+ public Map toRanges(int maxRanges, java.util.function.IntConsumer truncationNotifier) {
+ Map internalBitSetMap = new HashMap<>();
+ MutableInt rangeCount = new MutableInt();
+ MutableInt skippedLedgers = new MutableInt();
+ rangeBitmapMap.forEach((ledgerId, bitmap) -> {
+ if (rangeCount.addAndGet((int) bitmap.cardinality()) > maxRanges) {
+ skippedLedgers.increment();
+ return;
+ }
+ internalBitSetMap.put(ledgerId, bitmap.serializeToLongArray());
+ });
+ if (truncationNotifier != null && skippedLedgers.intValue() > 0) {
+ truncationNotifier.accept(skippedLedgers.intValue());
+ }
+ return internalBitSetMap;
+ }
+
+ @Override
+ public void build(Map internalRange) {
+ rangeBitmapMap.clear();
+ resetDirtyKeys();
+
+ internalRange.forEach((ledgerId, ranges) -> {
+ rangeBitmapMap.put(ledgerId.longValue(), LongBitmaps.deserializeFromLongArray(ranges));
+ });
+ recomputeTotalCardinality();
+ invalidateCaches();
+ }
+
+ void buildFromBitmaps(Map bitmaps) {
+ rangeBitmapMap.clear();
+ resetDirtyKeys();
+ bitmaps.forEach((ledgerId, bytes) -> {
+ if (bytes != null && bytes.length > 0) {
+ rangeBitmapMap.put(ledgerId.longValue(),
+ LongBitmaps.deserialize(Unpooled.wrappedBuffer(bytes)));
+ }
+ });
+ recomputeTotalCardinality();
+ invalidateCaches();
+ }
+
+ @Override
+ public int cardinality(long lowerLedgerId, long lowerEntryId, long upperLedgerId, long upperEntryId) {
+ Long2ObjectSortedMap subMap = rangeBitmapMap.subMap(lowerLedgerId, upperLedgerId + 1);
+ MutableInt v = new MutableInt(0);
+ subMap.forEach((ledgerId, bitmap) -> {
+ if (ledgerId == lowerLedgerId && ledgerId == upperLedgerId) {
+ long count = bitmap.rank(upperEntryId + 1) - bitmap.rank(lowerEntryId);
+ v.add(Math.toIntExact(count));
+ } else if (ledgerId == lowerLedgerId) {
+ long count = bitmap.cardinality() - bitmap.rank(lowerEntryId);
+ v.add(Math.toIntExact(count));
+ } else if (ledgerId == upperLedgerId) {
+ long count = bitmap.rank(upperEntryId + 1);
+ v.add(Math.toIntExact(count));
+ } else {
+ v.add(Math.toIntExact(bitmap.cardinality()));
+ }
+ });
+ return v.intValue();
+ }
+
+ @Override
+ public int size() {
+ if (updatedAfterCachedForSize) {
+ MutableInt size = new MutableInt(0);
+ forEachRawRange((lowerLedgerId, lowerEntryId, upperLedgerId, upperEntryId) -> {
+ size.increment();
+ return true;
+ });
+ cachedSize = size.intValue();
+ updatedAfterCachedForSize = false;
+ }
+ return cachedSize;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(rangeBitmapMap);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof PositionRangeSet other)) {
+ return false;
+ }
+ if (this == obj) {
+ return true;
+ }
+ return this.rangeBitmapMap.equals(other.rangeBitmapMap);
+ }
+
+ @Override
+ public String toString() {
+ if (updatedAfterCachedForToString) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("[");
+ AtomicBoolean first = new AtomicBoolean(true);
+ forEach(range -> {
+ if (!first.get()) {
+ sb.append(",");
+ }
+ sb.append(range);
+ first.set(false);
+ return true;
+ });
+ sb.append("]");
+ cachedToString = sb.toString();
+ updatedAfterCachedForToString = false;
+ }
+ return cachedToString;
+ }
+
+ @VisibleForTesting
+ void add(Range range) {
+ Position lowerEndpoint = range.hasLowerBound() ? range.lowerEndpoint()
+ : PositionFactory.create(EARLIEST_LEDGER_ID, EARLIEST_ENTRY_ID);
+ Position upperEndpoint = range.hasUpperBound() ? range.upperEndpoint()
+ : PositionFactory.create(LATEST_LEDGER_ID, LATEST_ENTRY_ID);
+
+ long lowerEntryIdOpen = (range.hasLowerBound() && range.lowerBoundType().equals(BoundType.CLOSED))
+ ? getSafeEntry(lowerEndpoint) - 1
+ : getSafeEntry(lowerEndpoint);
+ long upperEntryIdClosed = (range.hasUpperBound() && range.upperBoundType().equals(BoundType.CLOSED))
+ ? getSafeEntry(upperEndpoint)
+ : getSafeEntry(upperEndpoint) + 1;
+
+ LongBitmap bitmap = rangeBitmapMap.computeIfAbsent(lowerEndpoint.getLedgerId(),
+ k -> LongBitmaps.create());
+ if (bitmap.checkedAdd(lowerEntryIdOpen + 1)) {
+ totalCardinality++;
+ }
+ addOpenClosed(lowerEndpoint.getLedgerId(), lowerEntryIdOpen,
+ upperEndpoint.getLedgerId(), upperEntryIdClosed);
+ }
+
+ @VisibleForTesting
+ void remove(Range range) {
+ Position lowerEndpoint = range.hasLowerBound() ? range.lowerEndpoint()
+ : PositionFactory.create(EARLIEST_LEDGER_ID, EARLIEST_ENTRY_ID);
+ Position upperEndpoint = range.hasUpperBound() ? range.upperEndpoint()
+ : PositionFactory.create(LATEST_LEDGER_ID, LATEST_ENTRY_ID);
+
+ long lowerEntryId = (range.hasLowerBound() && range.lowerBoundType().equals(BoundType.CLOSED))
+ ? getSafeEntry(lowerEndpoint)
+ : getSafeEntry(lowerEndpoint) + 1;
+ long upperEntryId = (range.hasUpperBound() && range.upperBoundType().equals(BoundType.CLOSED))
+ ? getSafeEntry(upperEndpoint)
+ : getSafeEntry(upperEndpoint) - 1;
+
+ long lowerLedgerId = lowerEndpoint.getLedgerId();
+ long upperLedgerId = upperEndpoint.getLedgerId();
+ boolean lowerIsEarliest = lowerLedgerId == EARLIEST_LEDGER_ID
+ && lowerEndpoint.getEntryId() == EARLIEST_ENTRY_ID;
+ boolean upperIsLatest = upperLedgerId == LATEST_LEDGER_ID
+ && upperEndpoint.getEntryId() == LATEST_ENTRY_ID;
+ boolean sameLedger = lowerLedgerId == upperLedgerId;
+
+ if (lowerIsEarliest) {
+ clearSubMapCardinality(rangeBitmapMap.headMap(upperLedgerId));
+ }
+ if (upperIsLatest) {
+ clearSubMapCardinality(rangeBitmapMap.tailMap(lowerLedgerId + 1));
+ }
+ if (!sameLedger && !lowerIsEarliest && !upperIsLatest) {
+ clearSubMapCardinality(rangeBitmapMap.subMap(lowerLedgerId + 1, upperLedgerId));
+ }
+
+ LongBitmap lowerSet = lowerIsEarliest ? null : rangeBitmapMap.get(lowerLedgerId);
+ LongBitmap upperSet = upperIsLatest ? null
+ : (sameLedger ? lowerSet : rangeBitmapMap.get(upperLedgerId));
+
+ if (sameLedger && lowerSet != null) {
+ removeRange(lowerSet, lowerEntryId, upperEntryId + 1);
+ } else {
+ if (lowerSet != null) {
+ removeRange(lowerSet, lowerEntryId, lastPresentValue(lowerSet));
+ }
+ if (upperSet != null) {
+ removeRange(upperSet, 0, upperEntryId + 1);
+ }
+ }
+
+ if (lowerSet != null && lowerSet.isEmpty()) {
+ rangeBitmapMap.remove(lowerLedgerId);
+ }
+ if (!sameLedger && upperSet != null && upperSet.isEmpty()) {
+ rangeBitmapMap.remove(upperLedgerId);
+ }
+
+ invalidateCaches();
+ }
+
+ void resetDirtyKeys() {
+ dirtyLedgers.clear();
+ }
+
+ boolean isDirtyLedgers(long ledgerId) {
+ return ledgerId >= 0 && ledgerId <= Integer.MAX_VALUE && dirtyLedgers.contains(ledgerId);
+ }
+
+ /**
+ * Atomically snapshots the dirty ledger IDs and clears the dirty set.
+ *
+ * The returned set is consumed by the persistence layer to decide which ledgers
+ * need fresh AckState entries written this flush. Keys above {@code Integer.MAX_VALUE}
+ * are skipped at mark-time, so callers see raw ledger IDs in {@code [0, 2^31-1]}.
+ */
+ Set snapshotAndClearDirtyLedgers() {
+ Set snapshot = new HashSet<>();
+ dirtyLedgers.forEachLong(bit -> snapshot.add(bit - 1));
+ dirtyLedgers.clear();
+ return snapshot;
+ }
+
+ void restoreDirtyLedgers(Set ledgers) {
+ for (long id : ledgers) {
+ if (id >= 0 && id < Integer.MAX_VALUE) {
+ dirtyLedgers.add(id + 1);
+ }
+ }
+ }
+
+ /**
+ * Returns the serialized RoaringBitmap bytes for a specific ledger's individual acks,
+ * or {@code null} when the ledger has no entries in this set.
+ */
+ byte[] bitmapOf(long ledgerId) {
+ LongBitmap bitmap = rangeBitmapMap.get(ledgerId);
+ return bitmap == null ? null : bitmap.serialize();
+ }
+
+ /**
+ * Invokes the given consumer for each ledger ID that currently has at least one
+ * individual-deleted entry in this set. Used by the persistence layer to enumerate
+ * the active ledgers that need an AckState or AckStateRef in the checkpoint.
+ */
+ void forEachActiveLedger(LongConsumer action) {
+ rangeBitmapMap.keySet().forEach(action);
+ }
+
+ /**
+ * Marks a single ledger as dirty. Used for batch-index deletions, which are tracked in
+ * {@code batchDeletedIndexes} rather than in the individual-deleted range bitmap.
+ */
+ void markDirtyLedger(long ledgerId) {
+ markDirty(ledgerId, ledgerId);
+ }
+
+ private void markDirty(long lowerLedgerId, long upperLedgerId) {
+ if (upperLedgerId < lowerLedgerId || lowerLedgerId < 0) {
+ return;
+ }
+ if (lowerLedgerId >= Integer.MAX_VALUE || upperLedgerId > Integer.MAX_VALUE) {
+ log.warn("Skipping dirty tracking for ledger ID at/exceeding Integer.MAX_VALUE, "
+ + "lowerLedgerId: {}, upperLedgerId: {}",
+ lowerLedgerId, upperLedgerId);
+ return;
+ }
+ dirtyLedgers.add(lowerLedgerId + 1, upperLedgerId + 2);
+ }
+
+ private boolean isValid(long ledgerId, long entryId) {
+ return ledgerId != EARLIEST_LEDGER_ID && entryId != EARLIEST_ENTRY_ID
+ && ledgerId != LATEST_LEDGER_ID && entryId != LATEST_ENTRY_ID;
+ }
+
+ private int getSafeEntry(Position position) {
+ return getSafeEntry(position.getEntryId());
+ }
+
+ private int getSafeEntry(long value) {
+ return (int) Math.max(value, -1);
+ }
+
+ private void invalidateCaches() {
+ updatedAfterCachedForSize = true;
+ updatedAfterCachedForToString = true;
+ }
+
+ long totalCardinality() {
+ return totalCardinality;
+ }
+
+ private void recomputeTotalCardinality() {
+ long total = 0;
+ for (LongBitmap bitmap : rangeBitmapMap.values()) {
+ total += bitmap.cardinality();
+ }
+ totalCardinality = total;
+ }
+}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/RangeSetWrapper.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/RangeSetWrapper.java
deleted file mode 100644
index 76ac3e1be726c..0000000000000
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/RangeSetWrapper.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.mledger.impl;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Range;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import org.apache.pulsar.common.util.collections.LongPairRangeSet;
-import org.apache.pulsar.common.util.collections.OpenLongPairRangeSet;
-import org.roaringbitmap.RoaringBitSet;
-
-/**
- * Wraps other Range classes, and adds LRU, marking dirty data and other features on this basis.
- * This range set is not thread safety.
- *
- * @param
- */
-public class RangeSetWrapper> implements LongPairRangeSet {
-
- private final LongPairRangeSet rangeSet;
- private final LongPairConsumer rangeConverter;
- private final boolean enableMultiEntry;
-
- /**
- * Record which Ledger is dirty.
- */
- private final DefaultRangeSet dirtyLedgers = new LongPairRangeSet.DefaultRangeSet<>(
- (LongPairConsumer) (key, value) -> key,
- (RangeBoundConsumer) key -> new LongPair(key, 0));
-
- public RangeSetWrapper(LongPairConsumer rangeConverter,
- RangeBoundConsumer rangeBoundConsumer,
- ManagedCursorImpl managedCursor) {
- this(rangeConverter, rangeBoundConsumer, managedCursor.getConfig().isUnackedRangesOpenCacheSetEnabled(),
- managedCursor.getConfig().isPersistentUnackedRangesWithMultipleEntriesEnabled());
- }
-
- public RangeSetWrapper(LongPairConsumer rangeConverter,
- RangeBoundConsumer rangeBoundConsumer,
- boolean unackedRangesOpenCacheSetEnabled,
- boolean persistentUnackedRangesWithMultipleEntriesEnabled) {
- this.rangeConverter = rangeConverter;
- this.rangeSet = unackedRangesOpenCacheSetEnabled
- ? new OpenLongPairRangeSet<>(rangeConverter, RoaringBitSet::new)
- : new LongPairRangeSet.DefaultRangeSet<>(rangeConverter, rangeBoundConsumer);
- this.enableMultiEntry = persistentUnackedRangesWithMultipleEntriesEnabled;
- }
-
- @Override
- public void addOpenClosed(long lowerKey, long lowerValue, long upperKey, long upperValue) {
- if (enableMultiEntry) {
- dirtyLedgers.addOpenClosed(lowerKey, 0, upperKey, 0);
- }
- rangeSet.addOpenClosed(lowerKey, lowerValue, upperKey, upperValue);
- }
-
- @Override
- public boolean contains(long key, long value) {
- return rangeSet.contains(key, value);
- }
-
- @Override
- public Range rangeContaining(long key, long value) {
- return rangeSet.rangeContaining(key, value);
- }
-
- @Override
- public void removeAtMost(long key, long value) {
- if (enableMultiEntry) {
- dirtyLedgers.removeAtMost(key, 0);
- }
- rangeSet.removeAtMost(key, value);
- }
-
- @Override
- public boolean isEmpty() {
- return rangeSet.isEmpty();
- }
-
- @Override
- public void clear() {
- rangeSet.clear();
- dirtyLedgers.clear();
- }
-
- @Override
- public Range span() {
- return rangeSet.span();
- }
-
- @Override
- public Collection> asRanges() {
- Collection> collection = rangeSet.asRanges();
- if (collection instanceof List) {
- return collection;
- }
- return new ArrayList<>(collection);
- }
-
- @Override
- public void forEach(RangeProcessor action) {
- rangeSet.forEach(action);
- }
-
- @Override
- public void forEach(RangeProcessor action, LongPairConsumer consumer) {
- rangeSet.forEach(action, consumer);
- }
-
- @Override
- public void forEachRawRange(RawRangeProcessor action) {
- rangeSet.forEachRawRange(action);
- }
-
- @Override
- public int size() {
- return rangeSet.size();
- }
-
- @Override
- public Range firstRange() {
- return rangeSet.firstRange();
- }
-
- @Override
- public Range lastRange() {
- return rangeSet.lastRange();
- }
-
- @Override
- public Map toRanges(int maxRanges) {
- return rangeSet.toRanges(maxRanges);
- }
-
- @Override
- public void build(Map internalRange) {
- rangeSet.build(internalRange);
- }
-
- @Override
- public int cardinality(long lowerKey, long lowerValue, long upperKey, long upperValue) {
- return rangeSet.cardinality(lowerKey, lowerValue, upperKey, upperValue);
- }
-
- @VisibleForTesting
- void add(Range range) {
- if (!(rangeSet instanceof OpenLongPairRangeSet)) {
- throw new UnsupportedOperationException("Only ConcurrentOpenLongPairRangeSet support this method");
- }
- ((OpenLongPairRangeSet) rangeSet).add(range);
- }
-
- @VisibleForTesting
- void remove(Range range) {
- if (rangeSet instanceof OpenLongPairRangeSet) {
- ((OpenLongPairRangeSet) rangeSet).remove((Range) range);
- } else {
- ((DefaultRangeSet) rangeSet).remove(range);
- }
- }
-
- public void resetDirtyKeys() {
- dirtyLedgers.clear();
- }
-
- public boolean isDirtyLedgers(long ledgerId) {
- return dirtyLedgers.contains(ledgerId);
- }
-
- @Override
- public String toString() {
- return rangeSet.toString();
- }
-
- @Override
- public int hashCode() {
- return rangeSet.hashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- if (!(obj instanceof RangeSetWrapper)) {
- return false;
- }
- if (this == obj) {
- return true;
- }
- @SuppressWarnings("rawtypes")
- RangeSetWrapper set = (RangeSetWrapper) obj;
- return this.rangeSet.equals(set.rangeSet);
- }
-}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloadersCache.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloadersCache.java
index 9f2f5860d4ebd..03c87713566c4 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloadersCache.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloadersCache.java
@@ -19,6 +19,7 @@
package org.apache.bookkeeper.mledger.offload;
import java.io.IOException;
+import java.nio.file.Paths;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import lombok.extern.slf4j.Slf4j;
@@ -41,7 +42,8 @@ public class OffloadersCache implements AutoCloseable {
* @throws IOException when fail to retrieve the pulsar offloader class
*/
public Offloaders getOrLoadOffloaders(String offloadersPath, String narExtractionDirectory) {
- return loadedOffloaders.computeIfAbsent(offloadersPath,
+ String normalizedOffloadersPath = Paths.get(offloadersPath).toAbsolutePath().normalize().toString();
+ return loadedOffloaders.computeIfAbsent(normalizedOffloadersPath,
(directory) -> {
try {
return OffloaderUtils.searchForOffloaders(directory, narExtractionDirectory);
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/StatsBuckets.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/StatsBuckets.java
index 60c0a7f6c9d22..ec3a7e00fd390 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/StatsBuckets.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/StatsBuckets.java
@@ -60,6 +60,16 @@ public void addValue(long value) {
sumCounter.add(value);
}
+ /**
+ * Snapshots the current values into {@link #getBuckets()}, {@link #getCount()}, and
+ * {@link #getSum()}, then resets the internal counters to zero.
+ *
+ * Use this for periodic aggregation where the consumer reads values on a fixed
+ * schedule (e.g. {@code ManagedLedgerMBeanImpl} refreshes every
+ * {@code managedLedgerStatsPeriodSeconds}). Do not use this for metrics that
+ * are scraped on demand (e.g. Prometheus {@code /metrics}), because data recorded
+ * between scrapes is lost after each call — use {@link #snapshot()} instead.
+ */
public void refresh() {
long count = 0;
sum = sumCounter.sumThenReset();
@@ -73,6 +83,24 @@ public void refresh() {
this.count = count;
}
+ /**
+ * Snapshots the current cumulative values without resetting the internal counters.
+ * Use this instead of {@link #refresh()} when the data should survive across reads
+ * (e.g. Prometheus metrics that are scraped multiple times).
+ */
+ public void snapshot() {
+ long count = 0;
+ sum = sumCounter.sum();
+
+ for (int i = 0; i < buckets.length; i++) {
+ long value = buckets[i].sum();
+ count += value;
+ values[i] = value;
+ }
+
+ this.count = count;
+ }
+
public void reset() {
sum = 0;
sumCounter.reset();
diff --git a/managed-ledger/src/main/proto/MLDataFormats.proto b/managed-ledger/src/main/proto/MLDataFormats.proto
index 825da1e414cea..4109fcf36130a 100644
--- a/managed-ledger/src/main/proto/MLDataFormats.proto
+++ b/managed-ledger/src/main/proto/MLDataFormats.proto
@@ -140,6 +140,45 @@ message ManagedCursorInfo {
// Additional custom properties associated with
// the cursor
repeated StringProperty cursorProperties = 8;
+
+ optional int64 lastCmCursorLedgerId = 9;
+ optional int64 lastCmEntryId = 10;
+}
+
+message CursorLogEntry {
+ oneof entry {
+ CursorCheckpoint checkpoint = 2;
+ CursorCheckpointChunk checkpointChunk = 3;
+ }
+}
+
+// A consistent snapshot of cursor ack state for one persisted ledger update.
+// Each checkpoint entry contains exactly one inline ackState; other active
+// ledgers are represented by refs to their latest persisted checkpoints.
+message CursorCheckpoint {
+ required int64 markDeleteLedgerId = 1;
+ required int64 markDeleteEntryId = 2;
+ repeated LongProperty properties = 3;
+ repeated AckState ackStates = 4;
+ repeated AckStateRef ackStateRefs = 5;
+}
+
+message AckState {
+ required int64 msgLedgerId = 1;
+ optional bytes ackBitmap = 2;
+ repeated BatchedEntryDeletionIndexInfo batchAcks = 3;
+}
+
+message AckStateRef {
+ required int64 msgLedgerId = 1;
+ required int64 cursorLedgerId = 2;
+ required int64 entryId = 3;
+}
+
+message CursorCheckpointChunk {
+ optional int32 partIndex = 1;
+ optional int32 partCount = 2;
+ optional bytes checkpointBytes = 3;
}
enum CompressionType {
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/AutomaticOffloadTriggerControllerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/AutomaticOffloadTriggerControllerTest.java
new file mode 100644
index 0000000000000..9aefc7e3e1e1f
--- /dev/null
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/AutomaticOffloadTriggerControllerTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger.impl;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import org.testng.annotations.Test;
+
+public class AutomaticOffloadTriggerControllerTest {
+
+ @Test
+ public void triggersCoalesceWhileRunIsActive() {
+ AutomaticOffloadTriggerController controller = new AutomaticOffloadTriggerController();
+
+ assertThat(controller.requestRun()).isTrue();
+ assertThat(controller.requestRun()).isFalse();
+ assertThat(controller.requestRun()).isFalse();
+ }
+
+ @Test
+ public void pendingTriggerSchedulesOneFollowUpRun() {
+ AutomaticOffloadTriggerController controller = new AutomaticOffloadTriggerController();
+
+ assertThat(controller.requestRun()).isTrue();
+ assertThat(controller.requestRun()).isFalse();
+
+ assertThat(controller.completeRun()).isTrue();
+ assertThat(controller.completeRun()).isFalse();
+ assertThat(controller.requestRun()).isTrue();
+ }
+
+ @Test(timeOut = 30000)
+ public void concurrentTriggerAndCompletionAlwaysReserveOneFollowUpRun() throws Exception {
+ ExecutorService executor = Executors.newFixedThreadPool(2);
+ try {
+ for (int i = 0; i < 1000; i++) {
+ AutomaticOffloadTriggerController controller = new AutomaticOffloadTriggerController();
+ assertThat(controller.requestRun()).isTrue();
+
+ // Completion and a new trigger can race; exactly one side must reserve the follow-up run.
+ CyclicBarrier barrier = new CyclicBarrier(3);
+ Future completeResult = executor.submit(() -> {
+ barrier.await(5, TimeUnit.SECONDS);
+ return controller.completeRun();
+ });
+ Future triggerResult = executor.submit(() -> {
+ barrier.await(5, TimeUnit.SECONDS);
+ return controller.requestRun();
+ });
+
+ barrier.await(5, TimeUnit.SECONDS);
+ boolean followUpReservedByComplete = completeResult.get(5, TimeUnit.SECONDS);
+ boolean followUpReservedByTrigger = triggerResult.get(5, TimeUnit.SECONDS);
+
+ assertThat(followUpReservedByComplete)
+ .as("iteration %s must reserve exactly one follow-up run", i)
+ .isNotEqualTo(followUpReservedByTrigger);
+ assertThat(controller.completeRun()).isFalse();
+ }
+ } finally {
+ executor.shutdownNow();
+ executor.awaitTermination(5, TimeUnit.SECONDS);
+ }
+ }
+}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointLogRecoveryTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointLogRecoveryTest.java
new file mode 100644
index 0000000000000..e73f41fc930a3
--- /dev/null
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointLogRecoveryTest.java
@@ -0,0 +1,355 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger.impl;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import com.google.protobuf.ByteString;
+import java.util.concurrent.TimeUnit;
+import org.apache.bookkeeper.client.BookKeeper;
+import org.apache.bookkeeper.client.LedgerHandle;
+import org.apache.bookkeeper.mledger.ManagedLedgerException;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats.AckState;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats.CursorCheckpoint;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats.CursorCheckpointChunk;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats.CursorLogEntry;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats.PositionInfo;
+import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
+import org.testng.annotations.Test;
+
+public class CursorCheckpointLogRecoveryTest extends MockedBookKeeperTestCase {
+
+ private LedgerHandle createLedger() throws Exception {
+ return bkc.createLedger(BookKeeper.DigestType.MAC, new byte[0]);
+ }
+
+ private long appendRaw(LedgerHandle lh, byte[] data) throws Exception {
+ return lh.append(data);
+ }
+
+ private byte[] wrapCheckpoint(CursorCheckpoint cp) throws Exception {
+ return CursorLogEntry.newBuilder()
+ .setCheckpoint(CursorCheckpoint.parseFrom(cp.toByteArray()))
+ .build()
+ .toByteArray();
+ }
+
+ private byte[] wrapChunkPart(int partIndex, int partCount, byte[] payload) {
+ return CursorLogEntry.newBuilder()
+ .setCheckpointChunk(CursorCheckpointChunk.newBuilder()
+ .setPartIndex(partIndex).setPartCount(partCount)
+ .setCheckpointBytes(ByteString.copyFrom(payload))
+ .build())
+ .build()
+ .toByteArray();
+ }
+
+ private byte[] wrapLegacyPositionInfo(long ledgerId, long entryId) {
+ return PositionInfo.newBuilder()
+ .setLedgerId(ledgerId).setEntryId(entryId)
+ .build()
+ .toByteArray();
+ }
+
+ private CursorCheckpoint makeCheckpoint(long mdLedgerId, long mdEntryId) {
+ return CursorCheckpoint.newBuilder()
+ .setMarkDeleteLedgerId(mdLedgerId)
+ .setMarkDeleteEntryId(mdEntryId)
+ .addAckStates(AckState.newBuilder().setMsgLedgerId(mdLedgerId).build())
+ .build();
+ }
+
+ /**
+ * Scenario: checkpoint → legacy → chunk(complete) → chunk(incomplete) → chunk(incomplete)
+ *
+ * Recovery reads last (incomplete) → scanBack → finds chunk(complete) → assembles.
+ */
+ @Test(timeOut = 30000)
+ public void testScanBackFindsCompleteChunk() throws Exception {
+ LedgerHandle lh = createLedger();
+ CursorCheckpointLog writer = new CursorCheckpointLog(5 * 1024 * 1024);
+
+ appendRaw(lh, wrapCheckpoint(makeCheckpoint(10, 0)));
+ appendRaw(lh, wrapLegacyPositionInfo(10, 1));
+ // Write a complete 2-part chunk
+ byte[] cpBytes = makeCheckpoint(30, 5).toByteArray();
+ int half = cpBytes.length / 2;
+ byte[] part0 = new byte[half];
+ byte[] part1 = new byte[cpBytes.length - half];
+ System.arraycopy(cpBytes, 0, part0, 0, half);
+ System.arraycopy(cpBytes, half, part1, 0, part1.length);
+ appendRaw(lh, wrapChunkPart(0, 2, part0));
+ appendRaw(lh, wrapChunkPart(1, 2, part1));
+ // Two incomplete chunks after
+ appendRaw(lh, wrapChunkPart(0, 3, new byte[100]));
+ appendRaw(lh, wrapChunkPart(1, 3, new byte[100]));
+
+ CursorCheckpointLog.RecoveredState state = writer.readLatest(lh).get(5, TimeUnit.SECONDS);
+ assertThat(state.isLegacy()).isFalse();
+ assertThat(state.checkpoint.getMarkDeleteLedgerId()).isEqualTo(30L);
+ }
+
+ /**
+ * Scenario: checkpoint → legacy → chunk(incomplete)
+ *
+ * Recovery reads last (incomplete) → scanBack → finds legacy → returns legacy.
+ */
+ @Test(timeOut = 30000)
+ public void testScanBackFallsBackToLegacy() throws Exception {
+ LedgerHandle lh = createLedger();
+ CursorCheckpointLog writer = new CursorCheckpointLog(5 * 1024 * 1024);
+
+ appendRaw(lh, wrapCheckpoint(makeCheckpoint(10, 0)));
+ appendRaw(lh, wrapLegacyPositionInfo(20, 5));
+ // Incomplete chunk
+ appendRaw(lh, wrapChunkPart(0, 3, new byte[100]));
+
+ CursorCheckpointLog.RecoveredState state = writer.readLatest(lh).get(5, TimeUnit.SECONDS);
+ assertThat(state.isLegacy()).isTrue();
+ PositionInfo pi = PositionInfo.parseFrom(state.legacyBytes);
+ assertThat(pi.getLedgerId()).isEqualTo(20L);
+ assertThat(pi.getEntryId()).isEqualTo(5L);
+ }
+
+ /**
+ * Scenario: only incomplete chunks, scanBack exhausts, error.
+ */
+ @Test(timeOut = 30000)
+ public void testScanBackExhausts() throws Exception {
+ LedgerHandle lh = createLedger();
+ CursorCheckpointLog writer = new CursorCheckpointLog(5 * 1024 * 1024);
+
+ appendRaw(lh, wrapChunkPart(0, 3, new byte[100]));
+ appendRaw(lh, wrapChunkPart(1, 3, new byte[100]));
+
+ try {
+ writer.readLatest(lh).get(5, TimeUnit.SECONDS);
+ assertThat(false).as("Should have failed").isTrue();
+ } catch (Exception e) {
+ assertThat(e).hasCauseInstanceOf(ManagedLedgerException.class);
+ }
+ }
+
+ /**
+ * Scenario: last entry is legacy → returns directly (no scanBack).
+ */
+ @Test(timeOut = 30000)
+ public void testLegacyAsLastEntry() throws Exception {
+ LedgerHandle lh = createLedger();
+ CursorCheckpointLog writer = new CursorCheckpointLog(5 * 1024 * 1024);
+
+ appendRaw(lh, wrapCheckpoint(makeCheckpoint(10, 0)));
+ appendRaw(lh, wrapLegacyPositionInfo(20, 5));
+
+ CursorCheckpointLog.RecoveredState state = writer.readLatest(lh).get(5, TimeUnit.SECONDS);
+ assertThat(state.isLegacy()).isTrue();
+ }
+
+ /**
+ * Scenario: last entry is a valid checkpoint → returns directly.
+ */
+ @Test(timeOut = 30000)
+ public void testCheckpointAsLastEntry() throws Exception {
+ LedgerHandle lh = createLedger();
+ CursorCheckpointLog writer = new CursorCheckpointLog(5 * 1024 * 1024);
+
+ appendRaw(lh, wrapLegacyPositionInfo(10, 0));
+ appendRaw(lh, wrapCheckpoint(makeCheckpoint(20, 5)));
+
+ CursorCheckpointLog.RecoveredState state = writer.readLatest(lh).get(5, TimeUnit.SECONDS);
+ assertThat(state.isLegacy()).isFalse();
+ assertThat(state.checkpoint.getMarkDeleteLedgerId()).isEqualTo(20L);
+ }
+
+ /**
+ * Scenario: old, chunk(complete as last entry).
+ */
+ @Test(timeOut = 30000)
+ public void testCompleteChunkAsLastEntry() throws Exception {
+ LedgerHandle lh = createLedger();
+ CursorCheckpointLog writer = new CursorCheckpointLog(5 * 1024 * 1024);
+
+ appendRaw(lh, wrapLegacyPositionInfo(10, 0));
+ byte[] cpBytes = makeCheckpoint(40, 7).toByteArray();
+ int half = cpBytes.length / 2;
+ byte[] part0 = new byte[half];
+ byte[] part1 = new byte[cpBytes.length - half];
+ System.arraycopy(cpBytes, 0, part0, 0, half);
+ System.arraycopy(cpBytes, half, part1, 0, part1.length);
+ appendRaw(lh, wrapChunkPart(0, 2, part0));
+ appendRaw(lh, wrapChunkPart(1, 2, part1));
+
+ CursorCheckpointLog.RecoveredState state = writer.readLatest(lh).get(5, TimeUnit.SECONDS);
+ assertThat(state.isLegacy()).isFalse();
+ assertThat(state.checkpoint.getMarkDeleteLedgerId()).isEqualTo(40L);
+ }
+
+ /**
+ * Scenario: chunk(complete), chunk(incomplete).
+ */
+ @Test(timeOut = 30000)
+ public void testCompleteThenIncompleteChunkFallsBackToPreviousCompleteChunk() throws Exception {
+ LedgerHandle lh = createLedger();
+ CursorCheckpointLog writer = new CursorCheckpointLog(5 * 1024 * 1024);
+
+ byte[] cpBytes = makeCheckpoint(50, 9).toByteArray();
+ int half = cpBytes.length / 2;
+ byte[] part0 = new byte[half];
+ byte[] part1 = new byte[cpBytes.length - half];
+ System.arraycopy(cpBytes, 0, part0, 0, half);
+ System.arraycopy(cpBytes, half, part1, 0, part1.length);
+ appendRaw(lh, wrapChunkPart(0, 2, part0));
+ appendRaw(lh, wrapChunkPart(1, 2, part1));
+ appendRaw(lh, wrapChunkPart(0, 3, new byte[64]));
+
+ CursorCheckpointLog.RecoveredState state = writer.readLatest(lh).get(5, TimeUnit.SECONDS);
+ assertThat(state.isLegacy()).isFalse();
+ assertThat(state.checkpoint.getMarkDeleteLedgerId()).isEqualTo(50L);
+ }
+
+ /**
+ * Scenario: new, chunk(incomplete).
+ */
+ @Test(timeOut = 30000)
+ public void testNewThenIncompleteChunkFallsBackToNewCheckpoint() throws Exception {
+ LedgerHandle lh = createLedger();
+ CursorCheckpointLog writer = new CursorCheckpointLog(5 * 1024 * 1024);
+
+ appendRaw(lh, wrapCheckpoint(makeCheckpoint(60, 11)));
+ appendRaw(lh, wrapChunkPart(0, 3, new byte[64]));
+
+ CursorCheckpointLog.RecoveredState state = writer.readLatest(lh).get(5, TimeUnit.SECONDS);
+ assertThat(state.isLegacy()).isFalse();
+ assertThat(state.checkpoint.getMarkDeleteLedgerId()).isEqualTo(60L);
+ }
+
+ /**
+ * Boundary: recovery keeps scanning back, but only within MAX_SCAN_BACK (1000).
+ */
+ @Test(timeOut = 30000)
+ public void testScanBackLimitBoundary() throws Exception {
+ LedgerHandle lh = createLedger();
+ CursorCheckpointLog writer = new CursorCheckpointLog(5 * 1024 * 1024);
+
+ appendRaw(lh, wrapCheckpoint(makeCheckpoint(70, 13)));
+ for (int i = 0; i < 1000; i++) {
+ appendRaw(lh, wrapChunkPart(0, 3, new byte[16]));
+ }
+ CursorCheckpointLog.RecoveredState withinLimit = writer.readLatest(lh).get(5, TimeUnit.SECONDS);
+ assertThat(withinLimit.isLegacy()).isFalse();
+ assertThat(withinLimit.checkpoint.getMarkDeleteLedgerId()).isEqualTo(70L);
+
+ appendRaw(lh, wrapChunkPart(0, 3, new byte[16]));
+ try {
+ writer.readLatest(lh).get(5, TimeUnit.SECONDS);
+ assertThat(false).as("Should fail after exceeding scan-back limit").isTrue();
+ } catch (Exception e) {
+ assertThat(e).hasCauseInstanceOf(ManagedLedgerException.class);
+ }
+ }
+
+ /**
+ * Scenario: entry parses as CursorLogEntry but contains no known payload.
+ *
+ * Recovery should fail fast because this is a data error.
+ */
+ @Test(timeOut = 30000)
+ public void testInvalidEnvelopeFailsRecovery() throws Exception {
+ LedgerHandle lh = createLedger();
+ CursorCheckpointLog writer = new CursorCheckpointLog(5 * 1024 * 1024);
+
+ appendRaw(lh, CursorLogEntry.getDefaultInstance().toByteArray());
+
+ try {
+ writer.readLatest(lh).get(5, TimeUnit.SECONDS);
+ assertThat(false).as("Should have failed").isTrue();
+ } catch (Exception e) {
+ assertThat(e).hasCauseInstanceOf(ManagedLedgerException.class);
+ }
+ }
+
+ /**
+ * Scenario: bytes cannot be parsed as CursorLogEntry or PositionInfo.
+ *
+ * Recovery should fail rather than treating corrupted data as legacy.
+ */
+ @Test(timeOut = 30000)
+ public void testCorruptedBytesFailRecovery() throws Exception {
+ LedgerHandle lh = createLedger();
+ CursorCheckpointLog writer = new CursorCheckpointLog(5 * 1024 * 1024);
+
+ appendRaw(lh, new byte[]{(byte) 0x80});
+
+ try {
+ writer.readLatest(lh).get(5, TimeUnit.SECONDS);
+ assertThat(false).as("Should have failed").isTrue();
+ } catch (Exception e) {
+ assertThat(e).hasCauseInstanceOf(ManagedLedgerException.class);
+ }
+ }
+
+ /**
+ * Scenario: latest chunk looks complete, but assembly fails.
+ *
+ * Recovery should continue scan-back and return the previous valid checkpoint.
+ */
+ @Test(timeOut = 30000)
+ public void testBrokenCompleteChunkFallsBackToPreviousCheckpoint() throws Exception {
+ LedgerHandle lh = createLedger();
+ CursorCheckpointLog writer = new CursorCheckpointLog(5 * 1024 * 1024);
+
+ appendRaw(lh, wrapCheckpoint(makeCheckpoint(80, 15)));
+ appendRaw(lh, new byte[]{0x01, 0x02, 0x03}); // not a valid chunk part
+ appendRaw(lh, wrapChunkPart(1, 2, new byte[]{0x11, 0x22}));
+
+ CursorCheckpointLog.RecoveredState state = writer.readLatest(lh).get(5, TimeUnit.SECONDS);
+ assertThat(state.isLegacy()).isFalse();
+ assertThat(state.checkpoint.getMarkDeleteLedgerId()).isEqualTo(80L);
+ }
+
+ /**
+ * Scenario: readAt on the last part of a chunked checkpoint returns the assembled checkpoint,
+ * while readAt on a non-terminal part fails fast (ref targets must not scan back).
+ */
+ @Test(timeOut = 30000)
+ public void testReadAtAssemblesChunkAndFailsOnNonTerminalPart() throws Exception {
+ LedgerHandle lh = createLedger();
+ CursorCheckpointLog writer = new CursorCheckpointLog(5 * 1024 * 1024);
+
+ byte[] cpBytes = makeCheckpoint(90, 21).toByteArray();
+ int half = cpBytes.length / 2;
+ byte[] part0 = new byte[half];
+ byte[] part1 = new byte[cpBytes.length - half];
+ System.arraycopy(cpBytes, 0, part0, 0, half);
+ System.arraycopy(cpBytes, half, part1, 0, part1.length);
+ long firstEntry = appendRaw(lh, wrapChunkPart(0, 2, part0));
+ long lastEntry = appendRaw(lh, wrapChunkPart(1, 2, part1));
+
+ CursorCheckpointLog.RecoveredState state = writer.readAt(lh, lastEntry).get(5, TimeUnit.SECONDS);
+ assertThat(state.isLegacy()).isFalse();
+ assertThat(state.checkpoint.getMarkDeleteLedgerId()).isEqualTo(90L);
+ assertThat(state.commitEntryId).isEqualTo(lastEntry);
+
+ try {
+ writer.readAt(lh, firstEntry).get(5, TimeUnit.SECONDS);
+ assertThat(false).as("Should have failed on a non-terminal chunk part").isTrue();
+ } catch (Exception e) {
+ assertThat(e).hasCauseInstanceOf(ManagedLedgerException.class);
+ }
+ }
+}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointPersistenceOrderingTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointPersistenceOrderingTest.java
new file mode 100644
index 0000000000000..241a5dbab64c2
--- /dev/null
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/CursorCheckpointPersistenceOrderingTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger.impl;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNull;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.PositionFactory;
+import org.testng.annotations.Test;
+
+/**
+ * Ordering guarantees of {@link CursorCheckpointPersistence}: the per-msg-ledger checkpoint
+ * position index must never regress, otherwise a later AckStateRef would point at an older
+ * (smaller) ack bitmap and acks recorded in the newer checkpoint would be lost on recovery.
+ */
+public class CursorCheckpointPersistenceOrderingTest {
+
+ private CursorCheckpointPersistence newInstance() {
+ return new CursorCheckpointPersistence(null, new ReentrantReadWriteLock(),
+ null, null, new byte[0], true);
+ }
+
+ @Test
+ public void testRecordCheckpointPosMonotonic() {
+ CursorCheckpointPersistence persistence = newInstance();
+ assertNull(persistence.checkpointPosOf(5));
+
+ // First record wins by absence.
+ persistence.recordCheckpointPos(5, pos(1, 10));
+ assertEquals(persistence.checkpointPosOf(5), pos(1, 10));
+
+ // Newer position advances the index.
+ persistence.recordCheckpointPos(5, pos(1, 20));
+ assertEquals(persistence.checkpointPosOf(5), pos(1, 20));
+
+ // A stale completion (out-of-order append callback, same cursor ledger) must not regress.
+ persistence.recordCheckpointPos(5, pos(1, 15));
+ assertEquals(persistence.checkpointPosOf(5), pos(1, 20));
+
+ // A position from an older cursor ledger (after rollover replay) must not regress either.
+ persistence.recordCheckpointPos(5, pos(0, 999));
+ assertEquals(persistence.checkpointPosOf(5), pos(1, 20));
+
+ // Ledger ids are independent entries.
+ persistence.recordCheckpointPos(7, pos(1, 3));
+ assertEquals(persistence.checkpointPosOf(7), pos(1, 3));
+ assertEquals(persistence.checkpointPosOf(5), pos(1, 20));
+ }
+
+ private static Position pos(long ledgerId, long entryId) {
+ return PositionFactory.create(ledgerId, entryId);
+ }
+}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java
index f7b6e755bce83..1b8c999eef37a 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java
@@ -70,7 +70,7 @@ public void deleteFailed(ManagedLedgerException exception, Object ctx) {
@Test(dataProvider = "useOpenRangeSet")
public void testMarkDeleteAndRead(boolean useOpenRangeSet) throws Exception {
ManagedLedgerConfig config = new ManagedLedgerConfig().setMaxEntriesPerLedger(2)
- .setUnackedRangesOpenCacheSetEnabled(useOpenRangeSet);
+ ;
ManagedLedger ledger = factory.open("my_test_ledger", config);
final ManagedCursor cursor = ledger.openCursor("c1");
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorIndividualDeletedMessagesTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorIndividualDeletedMessagesTest.java
index b6c4cc5895db5..86e3bcbd22307 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorIndividualDeletedMessagesTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorIndividualDeletedMessagesTest.java
@@ -44,7 +44,6 @@ void testRecoverIndividualDeletedMessages() throws Exception {
BookKeeper bookkeeper = mock(BookKeeper.class);
ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setUnackedRangesOpenCacheSetEnabled(true);
NavigableMap ledgersInfo = new ConcurrentSkipListMap<>();
ledgersInfo.put(1L, createLedgerInfo(1, 100, 1024));
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorSkipDeletedEntriesTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorSkipDeletedEntriesTest.java
new file mode 100644
index 0000000000000..8c7dbd93dbf2d
--- /dev/null
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorSkipDeletedEntriesTest.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger.impl;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
+import lombok.Cleanup;
+import org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntriesCallback;
+import org.apache.bookkeeper.mledger.Entry;
+import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
+import org.apache.bookkeeper.mledger.ManagedLedgerException;
+import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.PositionFactory;
+import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
+import org.testng.annotations.Test;
+
+/**
+ * Regression test for reading over a cursor that has a very large contiguous range of individually
+ * deleted (acknowledged) entries while its mark-delete position is pinned by an older unacknowledged entry.
+ *
+ * This is the shape a Key_Shared subscription ends up in when a single unacknowledged message blocks the
+ * mark-delete position while consumers keep acknowledging everything after it. Reads then have to get
+ * past tens of millions of already-acknowledged entries before they can fill a batch.
+ *
+ *
The skip pre-filter in {@link ManagedLedgerImpl#internalReadFromLedger} used to advance the read
+ * position by the size of the scan window when every entry in that window was already acknowledged, instead of
+ * hopping over the whole deleted range via
+ * {@link ManagedCursorImpl#getNextAvailablePosition(Position)}. That turned a single range hop into
+ * O(entries / batchSize) read-loop iterations. Each iteration re-ran
+ * {@code isLedgerFullyAcked} -> {@code ManagedCursorImpl#getNumberOfEntries} ->
+ * {@code RangeSetWrapper#cardinality}, which clones the cursor's per-ledger Roaring bitmap.
+ */
+public class ManagedCursorSkipDeletedEntriesTest extends MockedBookKeeperTestCase {
+
+ private static final int TOTAL_ENTRIES = 20_000;
+ private static final int READ_BATCH_SIZE = 100;
+ /** Entries {@code [1, LAST_DELETED_INDEX]} are individually deleted, forming one contiguous range. */
+ private static final int LAST_DELETED_INDEX = TOTAL_ENTRIES - 4;
+
+ /**
+ * A read that has to get past one large contiguous deleted range must hop over it rather than walk
+ * it one batch at a time.
+ */
+ @Test(timeOut = 300_000)
+ public void testReadHopsOverLargeDeletedRangeInsteadOfWalkingIt() throws Exception {
+ ManagedLedgerConfig config = new ManagedLedgerConfig()
+ // Keep every entry below in a single ledger, then force a rollover so that the reads
+ // under test target a closed ledger. That makes every read-loop iteration go through
+ // the isLedgerFullyAcked check, as it does in production.
+ .setMaxEntriesPerLedger(TOTAL_ENTRIES)
+ .setRetentionTime(1, TimeUnit.HOURS)
+ .setRetentionSizeInMB(-1);
+
+ @Cleanup
+ ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("skip-deleted-entries", config);
+ ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("sub");
+
+ List positions = new ArrayList<>(TOTAL_ENTRIES);
+ for (int i = 0; i < TOTAL_ENTRIES; i++) {
+ positions.add(ledger.addEntry(new byte[]{1}));
+ }
+ // Roll the ledger holding the entries above, so it is closed by the time it is read.
+ Position afterRollover = ledger.addEntry(new byte[]{1});
+ assertTrue(ledger.getLedgersInfoAsList().size() >= 2,
+ "expected the entries under test to live in a closed ledger, ledgers="
+ + ledger.getLedgersInfoAsList().size());
+
+ // Individually acknowledge [1, LAST_DELETED_INDEX]. Entry 0 stays unacknowledged and pins the
+ // mark-delete position, so none of these acknowledgments can be collapsed into it.
+ cursor.delete(positions.subList(1, LAST_DELETED_INDEX + 1));
+
+ assertEquals(cursor.getMarkDeletedPosition(), PositionFactory.create(positions.get(0).getLedgerId(), -1),
+ "mark-delete position must stay pinned behind the unacknowledged entry 0");
+ assertEquals(cursor.getTotalNonContiguousDeletedMessagesRange(), 1,
+ "the acknowledgments must collapse into a single contiguous range");
+
+ // The deliverable entries are entry 0, the tail holes, and the entry that caused the rollover.
+ List expected = new ArrayList<>();
+ expected.add(positions.get(0));
+ for (int i = LAST_DELETED_INDEX + 1; i < TOTAL_ENTRIES; i++) {
+ expected.add(positions.get(i));
+ }
+ expected.add(afterRollover);
+
+ // Count every position the skip pre-filter examines. Predicate#or evaluates this one first, so
+ // it sees exactly the positions ManagedLedgerImpl walks before deciding what to read.
+ AtomicInteger examinedPositions = new AtomicInteger();
+ Predicate countingCondition = position -> {
+ examinedPositions.incrementAndGet();
+ return false;
+ };
+
+ CompletableFuture> readFuture = new CompletableFuture<>();
+ cursor.asyncReadEntriesWithSkip(READ_BATCH_SIZE, -1L, new ReadEntriesCallback() {
+ @Override
+ public void readEntriesComplete(List entries, Object ctx) {
+ readFuture.complete(entries);
+ }
+
+ @Override
+ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
+ readFuture.completeExceptionally(exception);
+ }
+ }, null, PositionFactory.LATEST, countingCondition);
+
+ List read = readFuture.get(120, TimeUnit.SECONDS);
+ List readPositions = new ArrayList<>();
+ for (Entry entry : read) {
+ readPositions.add(entry.getPosition());
+ entry.release();
+ }
+ assertEquals(readPositions, expected, "the read must return exactly the unacknowledged entries");
+
+ // Hopping the range examines one scan window before jumping it and a small number of windows for
+ // the entries actually returned. Walking it examines every position in the deleted block.
+ assertTrue(examinedPositions.get() <= 4 * READ_BATCH_SIZE,
+ "read examined " + examinedPositions.get() + " positions to return " + read.size()
+ + " entries over a deleted range of " + LAST_DELETED_INDEX + " entries");
+ }
+}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
index 72bd81c08b025..cdee6454d5df0 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
@@ -23,6 +23,7 @@
import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.nullable;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.eq;
@@ -89,11 +90,13 @@
import org.apache.bookkeeper.client.BookKeeper;
import org.apache.bookkeeper.client.BookKeeper.DigestType;
import org.apache.bookkeeper.client.LedgerEntry;
+import org.apache.bookkeeper.client.LedgerHandle;
import org.apache.bookkeeper.client.PulsarMockBookKeeper;
import org.apache.bookkeeper.client.PulsarMockReadHandleInterceptor;
import org.apache.bookkeeper.client.api.LedgerEntries;
import org.apache.bookkeeper.client.api.ReadHandle;
import org.apache.bookkeeper.common.util.OrderedExecutor;
+import org.apache.bookkeeper.common.util.OrderedScheduler;
import org.apache.bookkeeper.mledger.AsyncCallbacks;
import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback;
import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCallback;
@@ -124,6 +127,7 @@
import org.apache.commons.lang3.mutable.MutableBoolean;
import org.apache.pulsar.common.api.proto.CommandSubscribe;
import org.apache.pulsar.common.api.proto.IntRange;
+import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats;
import org.apache.pulsar.common.util.FutureUtil;
import org.apache.pulsar.common.util.collections.BitSetRecyclable;
import org.apache.pulsar.common.util.collections.LongPairRangeSet;
@@ -132,6 +136,7 @@
import org.apache.pulsar.metadata.api.extended.SessionEvent;
import org.apache.pulsar.metadata.impl.FaultInjectionMetadataStore;
import org.awaitility.Awaitility;
+import org.mockito.ArgumentCaptor;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
@@ -271,7 +276,7 @@ public void testConcurrentPropertyOperationsThreadSafety() throws Exception {
}
executor.shutdown();
- // Wait for each task to complete with timeout
+ // Wait for each task to complete with timeou
for (Future future : allFutures) {
try {
future.get(30, TimeUnit.SECONDS);
@@ -293,7 +298,7 @@ public void testConcurrentPropertyOperationsThreadSafety() throws Exception {
// 2. No inconsistent states detected
assertFalse(inconsistencyDetected.get(), "No inconsistent states (key with null value) should be detected");
- // 3: Final cursor state should be internally consistent
+ // 3: Final cursor state should be internally consisten
Map finalProperties = cursor.getProperties();
try {
for (Map.Entry entry : finalProperties.entrySet()) {
@@ -981,7 +986,7 @@ void testResetCursor1() throws Exception {
final AtomicBoolean moveStatus = new AtomicBoolean(false);
- // reset to earliest
+ // reset to earlies
Position earliest = PositionFactory.EARLIEST;
try {
cursor.resetCursor(earliest);
@@ -1551,7 +1556,7 @@ public void asyncMarkDeleteBlockingWithOneShot() throws Exception {
// just for log debug purpose
Deque positions = new ConcurrentLinkedDeque<>();
- // In previous flaky test, we set num=100, PR https://github.com/apache/pulsar/pull/25087 will make the test
+ // In previous flaky test, we set num=100, PR https://github.com/apache/pulsar/pull/25087 will make the tes
// more flaky. Flaky case:
// 1. cursor recovered with markDeletePosition 12:9, persistentMarkDeletePosition 12:9.
// 2. cursor recovered with mark markDeletePosition 13:-1, persistentMarkDeletePosition 13:-1.
@@ -2075,7 +2080,7 @@ void testCountingWithDeletedEntries() throws Exception {
@Test(timeOut = 20000, dataProvider = "useOpenRangeSet")
void testMarkDeleteTwice(boolean useOpenRangeSet) throws Exception {
ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig()
- .setUnackedRangesOpenCacheSetEnabled(useOpenRangeSet).setMaxEntriesPerLedger(2));
+ .setMaxEntriesPerLedger(2));
ManagedCursor cursor = ledger.openCursor("c1");
Position p1 = ledger.addEntry("entry1".getBytes());
@@ -2088,7 +2093,7 @@ void testMarkDeleteTwice(boolean useOpenRangeSet) throws Exception {
@Test(timeOut = 20000, dataProvider = "useOpenRangeSet")
void testSkipEntries(boolean useOpenRangeSet) throws Exception {
ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger", new ManagedLedgerConfig()
- .setUnackedRangesOpenCacheSetEnabled(useOpenRangeSet).setMaxEntriesPerLedger(2));
+ .setMaxEntriesPerLedger(2));
Position pos;
ManagedCursor c1 = ledger.openCursor("c1");
@@ -2138,7 +2143,7 @@ void testSkipEntries(boolean useOpenRangeSet) throws Exception {
@Test(timeOut = 20000, dataProvider = "useOpenRangeSet")
void testSkipEntriesWithIndividualDeletedMessages(boolean useOpenRangeSet) throws Exception {
ManagedLedger ledger = factory.open("testSkipEntriesWithIndividualDeletedMessages", new ManagedLedgerConfig()
- .setUnackedRangesOpenCacheSetEnabled(useOpenRangeSet).setMaxEntriesPerLedger(5));
+ .setMaxEntriesPerLedger(5));
ManagedCursor c1 = ledger.openCursor("c1");
Position pos1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding));
@@ -2186,7 +2191,7 @@ void testSkipEntriesWithIndividualDeletedMessages(boolean useOpenRangeSet) throw
@Test(timeOut = 20000, dataProvider = "useOpenRangeSet")
void testClearBacklog(boolean useOpenRangeSet) throws Exception {
ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig()
- .setUnackedRangesOpenCacheSetEnabled(useOpenRangeSet).setMaxEntriesPerLedger(1));
+ .setMaxEntriesPerLedger(1));
ManagedCursor c1 = ledger.openCursor("c1");
ledger.addEntry("dummy-entry-1".getBytes(Encoding));
@@ -2238,7 +2243,7 @@ void testClearBacklog(boolean useOpenRangeSet) throws Exception {
@Test(timeOut = 20000, dataProvider = "useOpenRangeSet")
void testRateLimitMarkDelete(boolean useOpenRangeSet) throws Exception {
ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setThrottleMarkDelete(1).setUnackedRangesOpenCacheSetEnabled(useOpenRangeSet); // Throttle to 1/s
+ config.setThrottleMarkDelete(1); // Throttle to 1/s
ManagedLedger ledger = factory.open("my_test_ledger", config);
ManagedCursor c1 = ledger.openCursor("c1");
@@ -2267,7 +2272,7 @@ void testRateLimitMarkDelete(boolean useOpenRangeSet) throws Exception {
@Test(timeOut = 20000, dataProvider = "useOpenRangeSet")
void deleteSingleMessageTwice(boolean useOpenRangeSet) throws Exception {
ManagedLedger ledger = factory.open("my_test_ledger",
- new ManagedLedgerConfig().setUnackedRangesOpenCacheSetEnabled(useOpenRangeSet));
+ new ManagedLedgerConfig());
ManagedCursor c1 = ledger.openCursor("c1");
@@ -2331,11 +2336,10 @@ void deleteSingleMessageTwice(boolean useOpenRangeSet) throws Exception {
assertEquals(c1.getMarkDeletedPosition(), p4);
assertEquals(c1.getReadPosition(), p4.getNext());
}
-
@Test(timeOut = 10000, dataProvider = "useOpenRangeSet")
void testReadEntriesOrWait(boolean useOpenRangeSet) throws Exception {
ManagedLedger ledger = factory.open("my_test_ledger",
- new ManagedLedgerConfig().setUnackedRangesOpenCacheSetEnabled(useOpenRangeSet));
+ new ManagedLedgerConfig());
final int consumers = 10;
final CountDownLatch counter = new CountDownLatch(consumers);
@@ -2481,7 +2485,7 @@ void testScan(int numEntries, int batchSize) throws Exception {
return true;
}), batchSize, 1, Long.MAX_VALUE).get());
- // timeout
+ // timeou
// please note that the timeout is verified
// between the reads
// so with a big batchSize this test would take too much
@@ -2750,7 +2754,7 @@ void testFindNewestMatchingEdgeCase5() throws Exception {
@Test(timeOut = 20000, dataProvider = "useOpenRangeSet")
void testFindNewestMatchingEdgeCase6(boolean useOpenRangeSet) throws Exception {
ManagedLedger ledger = factory.open("testFindNewestMatchingEdgeCase6", new ManagedLedgerConfig()
- .setUnackedRangesOpenCacheSetEnabled(useOpenRangeSet).setMaxEntriesPerLedger(3));
+ .setMaxEntriesPerLedger(3));
ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
ledger.addEntry("expired".getBytes(Encoding));
@@ -2769,7 +2773,7 @@ void testFindNewestMatchingEdgeCase6(boolean useOpenRangeSet) throws Exception {
@Test(timeOut = 20000, dataProvider = "useOpenRangeSet")
void testFindNewestMatchingEdgeCase7(boolean useOpenRangeSet) throws Exception {
ManagedLedger ledger = factory.open("testFindNewestMatchingEdgeCase7",
- new ManagedLedgerConfig().setUnackedRangesOpenCacheSetEnabled(useOpenRangeSet));
+ new ManagedLedgerConfig());
ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
ledger.addEntry("expired".getBytes(Encoding));
@@ -2864,7 +2868,7 @@ void testFindNewestMatchingEdgeCase10() throws Exception {
@Test(timeOut = 20000, dataProvider = "useOpenRangeSet")
void testIndividuallyDeletedMessages(boolean useOpenRangeSet) throws Exception {
ManagedLedger ledger = factory.open("testIndividuallyDeletedMessages",
- new ManagedLedgerConfig().setUnackedRangesOpenCacheSetEnabled(useOpenRangeSet));
+ new ManagedLedgerConfig());
ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
ledger.addEntry("entry-0".getBytes(Encoding));
@@ -2904,7 +2908,7 @@ void testIndividuallyDeletedMessages1() throws Exception {
@Test(timeOut = 20000, dataProvider = "useOpenRangeSet")
void testIndividuallyDeletedMessages2(boolean useOpenRangeSet) throws Exception {
ManagedLedger ledger = factory.open("testIndividuallyDeletedMessages2",
- new ManagedLedgerConfig().setUnackedRangesOpenCacheSetEnabled(useOpenRangeSet));
+ new ManagedLedgerConfig());
ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
ledger.addEntry("entry-0".getBytes(Encoding));
@@ -2925,7 +2929,7 @@ void testIndividuallyDeletedMessages2(boolean useOpenRangeSet) throws Exception
@Test(timeOut = 20000, dataProvider = "useOpenRangeSet")
void testIndividuallyDeletedMessages3(boolean useOpenRangeSet) throws Exception {
ManagedLedger ledger = factory.open("testIndividuallyDeletedMessages3",
- new ManagedLedgerConfig().setUnackedRangesOpenCacheSetEnabled(useOpenRangeSet));
+ new ManagedLedgerConfig());
ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
ledger.addEntry("entry-0".getBytes(Encoding));
@@ -2969,7 +2973,7 @@ void testFindNewestMatchingAfterLedgerRollover() throws Exception {
// then we are done
// there was a bug (https://github.com/apache/pulsar/issues/9082)
// in which if the last message was in a different ledger
- // the jump from the first message to the last message went
+ // the jump from the first message to the last message wen
// to an invalid position and so the search stopped at the first message
// we want to assert here that the algorithm returns the position of the
@@ -3112,7 +3116,7 @@ void testReplayEntries() throws Exception {
ledger.addEntry("entry4".getBytes(Encoding));
// 1. Replay empty position set should return empty entry set
- Set positions = new HashSet();
+ Set positions = new HashSet<>();
assertTrue(c1.replayEntries(positions).isEmpty());
positions.add(p1);
@@ -3535,7 +3539,7 @@ public void operationFailed(ManagedLedgerException exception) {
String path = "/managed-ledgers/my_test_ledger/c1";
metadataStore.put(path, "".getBytes(), Optional.empty()).join();
- // try to create ledger again which will fail because managedCursorInfo znode is already updated with different
+ // try to create ledger again which will fail because managedCursorInfo znode is already updated with differen
// version so, this call will fail with BadVersionException
CountDownLatch latch2 = new CountDownLatch(1);
// create ledger will create ledgerId = 6
@@ -3797,6 +3801,115 @@ public void testEstimatedUnackedSize() throws Exception {
assertEquals(cursor.getEstimatedSizeSinceMarkDeletePosition(), 10 * entryData.length);
}
+ @Test
+ public void testEstimatedUnackedSizeWhenLatestLedgerIsEmpty() throws Exception {
+ ManagedLedger ledger = factory.open("test_estimated_unacked_size_empty_latest_ledger");
+ ManagedCursor cursor = ledger.openCursor("c1");
+
+ assertEquals(cursor.getEstimatedSizeSinceMarkDeletePosition(), 0);
+ }
+
+ @Test
+ public void testEstimatedUnackedSizeWhenCursorCaughtUpWithLastPosition() throws Exception {
+ ManagedLedger ledger = factory.open("test_estimated_unacked_size_cursor_caught_up");
+ ManagedCursor cursor = ledger.openCursor("c1");
+
+ Position lastPosition = ledger.addEntry("entry".getBytes(Encoding));
+ cursor.markDelete(lastPosition);
+
+ assertEquals(cursor.getEstimatedSizeSinceMarkDeletePosition(), 0);
+ }
+
+ @Test
+ public void testScheduleReadCallbackUsesManagedLedgerExecutionContext() {
+ ManagedLedgerImpl ledger = mock(ManagedLedgerImpl.class);
+ when(ledger.getConfig()).thenReturn(new ManagedLedgerConfig());
+ OrderedScheduler scheduledExecutor = mock(OrderedScheduler.class);
+ ExecutorService executor = mock(ExecutorService.class);
+ when(ledger.getScheduledExecutor()).thenReturn(scheduledExecutor);
+ when(ledger.getExecutor()).thenReturn(executor);
+ ManagedCursorImpl cursor = new ManagedCursorImpl(mock(BookKeeper.class), ledger, "c1");
+ Runnable callback = mock(Runnable.class);
+ ArgumentCaptor scheduledTask = ArgumentCaptor.forClass(Runnable.class);
+
+ cursor.scheduleReadCallback(callback, 100, TimeUnit.MILLISECONDS);
+
+ verify(scheduledExecutor).schedule(scheduledTask.capture(), eq(100L), eq(TimeUnit.MILLISECONDS));
+ scheduledTask.getValue().run();
+ verify(executor).execute(callback);
+ }
+
+ @Test
+ public void testDefaultScheduleReadCallback() throws InterruptedException {
+ ManagedCursor cursor = mock(ManagedCursor.class, Mockito.CALLS_REAL_METHODS);
+ CountDownLatch callbackExecuted = new CountDownLatch(1);
+
+ cursor.scheduleReadCallback(callbackExecuted::countDown, 0, TimeUnit.MILLISECONDS);
+
+ assertTrue(callbackExecuted.await(5, TimeUnit.SECONDS));
+ }
+
+ @Test
+ public void testEstimatedUnackedSizeWhenCursorAdvancedToEmptyCurrentLedger() {
+ ManagedLedgerImpl ledger = mock(ManagedLedgerImpl.class);
+ when(ledger.getName()).thenReturn("test_estimated_unacked_size_empty_current_ledger");
+ when(ledger.getConfig()).thenReturn(new ManagedLedgerConfig());
+
+ long currentLedgerId = 4;
+ Position lastPosition = PositionFactory.create(3, 9);
+ Position markDeletePosition = PositionFactory.create(currentLedgerId, -1);
+ when(ledger.getLastPosition()).thenReturn(lastPosition);
+ when(ledger.ledgerExists(lastPosition.getLedgerId())).thenReturn(true);
+
+ LedgerHandle currentLedger = mock(LedgerHandle.class);
+ when(currentLedger.getId()).thenReturn(currentLedgerId);
+ ledger.currentLedger = currentLedger;
+ ledger.currentLedgerEntries = 0;
+
+ ManagedCursorImpl cursor = new ManagedCursorImpl(mock(BookKeeper.class), ledger, "c1");
+ cursor.markDeletePosition = markDeletePosition;
+
+ assertEquals(cursor.getEstimatedSizeSinceMarkDeletePosition(), 0);
+ verify(ledger, never()).estimateBacklogFromPosition(any());
+ }
+
+ @Test
+ public void testEstimatedUnackedSizeWhenLastPositionLedgerIsNoLongerInLedgerList() {
+ ManagedLedgerImpl ledger = mock(ManagedLedgerImpl.class);
+ when(ledger.getName()).thenReturn("test_estimated_unacked_size_last_position_ledger_removed");
+ when(ledger.getConfig()).thenReturn(new ManagedLedgerConfig());
+
+ Position lastPosition = PositionFactory.create(3, 0);
+ Position markDeletePosition = PositionFactory.create(4, -1);
+ when(ledger.getLastPosition()).thenReturn(lastPosition);
+ when(ledger.ledgerExists(lastPosition.getLedgerId())).thenReturn(false);
+
+ ManagedCursorImpl cursor = new ManagedCursorImpl(mock(BookKeeper.class), ledger, "c1");
+ cursor.markDeletePosition = markDeletePosition;
+
+ assertEquals(cursor.getEstimatedSizeSinceMarkDeletePosition(), 0);
+ }
+
+ @Test
+ public void testEstimatedUnackedSizeFailsWhenCursorIsUnexpectedlyAheadOfLastPosition() {
+ ManagedLedgerImpl ledger = mock(ManagedLedgerImpl.class);
+ when(ledger.getName()).thenReturn("test_estimated_unacked_size_unexpected_position");
+ when(ledger.getConfig()).thenReturn(new ManagedLedgerConfig());
+
+ Position lastPosition = PositionFactory.create(1, 10);
+ Position markDeletePosition = PositionFactory.create(2, -1);
+ when(ledger.getLastPosition()).thenReturn(lastPosition);
+ when(ledger.ledgerExists(lastPosition.getLedgerId())).thenReturn(true);
+ when(ledger.ledgerExists(markDeletePosition.getLedgerId())).thenReturn(true);
+
+ ManagedCursorImpl cursor = new ManagedCursorImpl(mock(BookKeeper.class), ledger, "c1");
+ cursor.markDeletePosition = markDeletePosition;
+
+ IllegalArgumentException exception = Assert.expectThrows(IllegalArgumentException.class,
+ cursor::getEstimatedSizeSinceMarkDeletePosition);
+ assertTrue(exception.getMessage().contains("is ahead of the last position"));
+ }
+
/**
* Test that cursor.getEstimatedSizeSinceMarkDeletePosition() correctly accounts for individual
* message deletions (asyncDelete/individual ack).
@@ -3866,7 +3979,7 @@ public void deleteFailed(ManagedLedgerException exception, Object ctx) {
assertEquals(actualBacklogSize, expectedBacklogSize,
"Backlog size should account for individual deletions");
- // Verify both count and size are correct
+ // Verify both count and size are correc
assertEquals(cursor.getNumberOfEntriesInBacklog(true), 2, "Backlog count should be 2");
ledger.close();
@@ -4704,7 +4817,7 @@ public void testCursorNoRolloverIfNoMetadataSession() throws Exception {
assertEquals(cursor.getCursorLedger(), initialLedgerId);
- // After the session gets reestablished, the rollover should restart
+ // After the session gets reestablished, the rollover should restar
metadataStore.triggerSessionEvent(SessionEvent.SessionReestablished);
for (int i = 0; i < 10; i++) {
@@ -5341,7 +5454,7 @@ public void findEntryFailed(ManagedLedgerException exception,
assertNotNull(positionRef.get());
assertEquals(positionRef.get(), position1);
- // find the newest entry with start
+ // find the newest entry with star
AtomicBoolean failed1 = new AtomicBoolean(false);
CountDownLatch latch1 = new CountDownLatch(1);
AtomicReference positionRef1 = new AtomicReference<>();
@@ -5498,7 +5611,7 @@ public void findEntryFailed(ManagedLedgerException exception,
assertNotNull(positionRef.get());
assertEquals(positionRef.get(), position3);
- // find the newest entry with start
+ // find the newest entry with star
AtomicBoolean failed1 = new AtomicBoolean(false);
CountDownLatch latch1 = new CountDownLatch(1);
AtomicReference positionRef1 = new AtomicReference<>();
@@ -6017,6 +6130,89 @@ public void deleteFailed(ManagedLedgerException exception, Object ctx) {
assertEquals(properties.get(propertyKey), lastIndex - 1);
}
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testCompactionCursorResetNeverLoseMarkDeleteProperties() throws Exception {
+ @Cleanup
+ ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open(
+ "testCompactionCursorResetNeverLoseMarkDeleteProperties",
+ new ManagedLedgerConfig().setMaxEntriesPerLedger(10));
+ ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("__compaction");
+ ManagedCursorImpl spyCursor = spy(cursor);
+ ledger.getCursors().removeCursor(cursor.getName());
+ ledger.getCursors().add(spyCursor, null);
+
+ ledger.addEntry("entry-1".getBytes(Encoding));
+ Position markDeletePosition = ledger.addEntry("entry-2".getBytes(Encoding));
+
+ String compactedLedgerProperty = "CompactedTopicLedger";
+ Map properties = Map.of(compactedLedgerProperty, 123456L);
+
+ CountDownLatch markDeleteEntered = new CountDownLatch(1);
+ CountDownLatch resetEntered = new CountDownLatch(1);
+ CountDownLatch markDeleteReturned = new CountDownLatch(1);
+ CountDownLatch markDeleteCompleted = new CountDownLatch(1);
+ CountDownLatch resetCompleted = new CountDownLatch(1);
+
+ doAnswer(invocation -> {
+ Map invocationProperties = invocation.getArgument(1);
+ if (invocationProperties != null && invocationProperties.containsKey(compactedLedgerProperty)) {
+ // Hold the compaction mark-delete after it enters internalAsyncMarkDelete, but before its
+ // properties can update lastMarkDeleteEntry.
+ markDeleteEntered.countDown();
+ assertTrue(resetEntered.await(5, TimeUnit.SECONDS));
+ try {
+ return invocation.callRealMethod();
+ } finally {
+ markDeleteReturned.countDown();
+ }
+ }
+
+ if (invocationProperties == null || invocationProperties.isEmpty()) {
+ // Let reset capture its properties argument first, then persist it only after the compaction
+ // mark-delete has completed the real internalAsyncMarkDelete call.
+ resetEntered.countDown();
+ assertTrue(markDeleteReturned.await(5, TimeUnit.SECONDS));
+ return invocation.callRealMethod();
+ }
+
+ return invocation.callRealMethod();
+ }).when(spyCursor).internalAsyncMarkDelete(any(Position.class), nullable(Map.class),
+ any(MarkDeleteCallback.class), nullable(Object.class), nullable(Runnable.class));
+
+ // Start compaction mark-delete from another thread because the spy intentionally blocks it.
+ CompletableFuture.runAsync(() -> spyCursor.asyncMarkDelete(
+ markDeletePosition, properties, new MarkDeleteCallback() {
+ @Override
+ public void markDeleteComplete(Object ctx) {
+ markDeleteCompleted.countDown();
+ }
+
+ @Override
+ public void markDeleteFailed(ManagedLedgerException exception, Object ctx) {
+ }
+ }, null));
+
+ assertTrue(markDeleteEntered.await(5, TimeUnit.SECONDS));
+ // Reset the compaction cursor while the previous mark-delete with properties is still in progress.
+ spyCursor.asyncResetCursor(markDeletePosition, false, new AsyncCallbacks.ResetCursorCallback() {
+ @Override
+ public void resetComplete(Object ctx) {
+ resetCompleted.countDown();
+ }
+
+ @Override
+ public void resetFailed(ManagedLedgerException exception, Object ctx) {
+ }
+ });
+
+ assertTrue(markDeleteCompleted.await(5, TimeUnit.SECONDS));
+ assertTrue(resetCompleted.await(5, TimeUnit.SECONDS));
+
+ assertEquals(spyCursor.getMarkDeletedPosition(), markDeletePosition);
+ assertEquals(spyCursor.getProperties(), properties);
+ }
+
class TestPulsarMockBookKeeper extends PulsarMockBookKeeper {
Map ledgerErrors = new HashMap<>();
@@ -6048,4 +6244,1265 @@ public void asyncOpenLedger(final long lId, final DigestType digestType, final b
}
private static final Logger log = LoggerFactory.getLogger(ManagedCursorTest.class);
+ @Test(timeOut = 30000)
+ public void testCheckpointIndividualAckPersistAndRecover() throws Exception {
+ ManagedLedgerConfig config = new ManagedLedgerConfig();
+ config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true);
+ config.setMaxUnackedRangesToPersistInMetadataStore(-1);
+ config.setThrottleMarkDelete(0);
+
+ String ledgerName = "test_checkpoint_individual_ack";
+ ManagedLedger ledger = factory.open(ledgerName, config);
+ ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1");
+ List positions = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ positions.add(ledger.addEntry(("msg-" + i).getBytes(Encoding)));
+ }
+
+ cursor.markDelete(positions.get(0));
+ cursor.delete(positions.get(2));
+ cursor.delete(positions.get(5));
+
+ assertThat(cursor.isMessageDeleted(positions.get(2))).isTrue();
+ assertThat(cursor.isMessageDeleted(positions.get(5))).isTrue();
+ assertThat(cursor.isMessageDeleted(positions.get(3))).isFalse();
+
+ ManagedCursorImpl cursorBeforeClose = cursor;
+ Awaitility.await().untilAsserted(
+ () -> assertThat(cursorBeforeClose.getCursorLedger()).isNotNegative());
+ ledger.close();
+
+ ledger = factory.open(ledgerName, config);
+ cursor = (ManagedCursorImpl) ledger.openCursor("c1");
+ assertThat(cursor.getMarkDeletedPosition()).isEqualTo(positions.get(0));
+ assertThat(cursor.isMessageDeleted(positions.get(2))).isTrue();
+ assertThat(cursor.isMessageDeleted(positions.get(5))).isTrue();
+ assertThat(cursor.isMessageDeleted(positions.get(3))).isFalse();
+ ledger.close();
+ }
+
+ @Test(timeOut = 30000)
+ public void testCheckpointMultiMsgLedgerRecovery() throws Exception {
+ ManagedLedgerConfig config = new ManagedLedgerConfig();
+ config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true);
+ config.setMaxEntriesPerLedger(5); // small msg-ledger → multiple msgLedgers
+ config.setMaxUnackedRangesToPersistInMetadataStore(-1);
+ config.setThrottleMarkDelete(0);
+
+ String ledgerName = "test_checkpoint_multi_ledger";
+ ManagedLedger ledger = factory.open(ledgerName, config);
+ ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1");
+ List positions = new ArrayList<>();
+ for (int i = 0; i < 20; i++) {
+ positions.add(ledger.addEntry(("msg-" + i).getBytes(Encoding)));
+ }
+
+ cursor.markDelete(positions.get(0));
+ cursor.delete(positions.get(3));
+ cursor.delete(positions.get(7));
+ cursor.delete(positions.get(12));
+ cursor.delete(positions.get(18));
+
+ ManagedCursorImpl cursorBeforeClose2 = cursor;
+ Awaitility.await().untilAsserted(
+ () -> assertThat(cursorBeforeClose2.getCursorLedger()).isNotNegative());
+ ledger.close();
+
+ ledger = factory.open(ledgerName, config);
+ cursor = (ManagedCursorImpl) ledger.openCursor("c1");
+ assertThat(cursor.getMarkDeletedPosition()).isEqualTo(positions.get(0));
+ assertThat(cursor.isMessageDeleted(positions.get(3))).isTrue();
+ assertThat(cursor.isMessageDeleted(positions.get(7))).isTrue();
+ assertThat(cursor.isMessageDeleted(positions.get(12))).isTrue();
+ assertThat(cursor.isMessageDeleted(positions.get(18))).isTrue();
+ assertThat(cursor.isMessageDeleted(positions.get(5))).isFalse();
+ ledger.close();
+ }
+
+ @Test(timeOut = 30000)
+ public void testCheckpointCursorLedgerRollover() throws Exception {
+ ManagedLedgerConfig config = new ManagedLedgerConfig();
+ config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true);
+ config.setMaxEntriesPerLedger(5);
+ config.setMetadataMaxEntriesPerLedger(5); // force frequent cursor-ledger rollover
+ config.setMaxUnackedRangesToPersistInMetadataStore(-1);
+ config.setThrottleMarkDelete(0);
+
+ String ledgerName = "test_checkpoint_rollover";
+ ManagedLedger ledger = factory.open(ledgerName, config);
+ ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1");
+
+ List positions = new ArrayList<>();
+ for (int i = 0; i < 30; i++) {
+ positions.add(ledger.addEntry(("msg-" + i).getBytes(Encoding)));
+ cursor.markDelete(positions.get(i));
+ }
+ cursor.delete(positions.get(5));
+ cursor.delete(positions.get(15));
+ cursor.delete(positions.get(25));
+
+ ManagedCursorImpl cursorBeforeClose3 = cursor;
+ Awaitility.await().untilAsserted(
+ () -> assertThat(cursorBeforeClose3.getCursorLedger()).isNotNegative());
+ ledger.close();
+
+ ledger = factory.open(ledgerName, config);
+ cursor = (ManagedCursorImpl) ledger.openCursor("c1");
+ assertThat(cursor.isMessageDeleted(positions.get(5))).isTrue();
+ assertThat(cursor.isMessageDeleted(positions.get(15))).isTrue();
+ assertThat(cursor.isMessageDeleted(positions.get(25))).isTrue();
+ ledger.close();
+ }
+
+ @Test(timeOut = 60000)
+ public void testCheckpointImmediateCloseFlushesAllAppliedIndividualAcks() throws Exception {
+ ManagedLedgerConfig config = new ManagedLedgerConfig();
+ config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true);
+ config.setMaxUnackedRangesToPersistInMetadataStore(-1);
+ config.setThrottleMarkDelete(0);
+
+ String ledgerName = "test_checkpoint_immediate_close";
+ ManagedLedger ledger = factory.open(ledgerName, config);
+ ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1");
+
+ List positions = new ArrayList<>();
+ for (int i = 0; i < 1000; i++) {
+ positions.add(ledger.addEntry(("msg-" + i).getBytes(Encoding)));
+ }
+ cursor.markDelete(positions.get(0));
+ // Individual-delete even positions without waiting for any periodic flush.
+ for (int i = 2; i < 1000; i += 2) {
+ cursor.delete(positions.get(i));
+ }
+
+ // Close immediately: no Awaitility wait for the periodic flush. All acks already
+ // applied to individualDeletedMessages must be captured by the closing persist.
+ ledger.close();
+
+ ledger = factory.open(ledgerName, config);
+ cursor = (ManagedCursorImpl) ledger.openCursor("c1");
+ assertThat(cursor.getMarkDeletedPosition()).isEqualTo(positions.get(0));
+ int lost = 0;
+ for (int i = 2; i < 1000; i += 2) {
+ if (!cursor.isMessageDeleted(positions.get(i))) {
+ lost++;
+ }
+ }
+ assertThat(lost).isZero();
+ for (int i = 1; i < 1000; i += 2) {
+ assertThat(cursor.isMessageDeleted(positions.get(i))).isFalse();
+ }
+ ledger.close();
+ }
+
+ @Test(timeOut = 60000)
+ public void testCheckpointChunkedPersistAndRecoverLargeAckSet() throws Exception {
+ ManagedLedgerConfig config = new ManagedLedgerConfig();
+ config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true);
+ // Force the checkpoint log to chunk entries: a single per-ledger bitmap of
+ // 5k alternating positions (~10KB Roaring) far exceeds this cap.
+ config.setPersistentUnackedRangesMaxEntrySize(2048);
+ config.setMaxUnackedRangesToPersistInMetadataStore(-1);
+ config.setThrottleMarkDelete(0);
+
+ String ledgerName = "test_checkpoint_chunked_large_ack_set";
+ ManagedLedger ledger = factory.open(ledgerName, config);
+ ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1");
+
+ final int total = 5_000;
+ List positions = new ArrayList<>(total);
+ for (int i = 0; i < total; i++) {
+ positions.add(ledger.addEntry(("msg-" + i).getBytes(Encoding)));
+ }
+ cursor.markDelete(positions.get(0));
+ for (int i = 2; i < total; i += 2) {
+ cursor.delete(positions.get(i));
+ }
+
+ ledger.close();
+
+ ledger = factory.open(ledgerName, config);
+ cursor = (ManagedCursorImpl) ledger.openCursor("c1");
+ assertThat(cursor.getMarkDeletedPosition()).isEqualTo(positions.get(0));
+ int lost = 0;
+ for (int i = 2; i < total; i += 2) {
+ if (!cursor.isMessageDeleted(positions.get(i))) {
+ lost++;
+ }
+ }
+ assertThat(lost).isZero();
+ assertThat(cursor.isMessageDeleted(positions.get(1))).isFalse();
+ ledger.close();
+ }
+
+ @Test(timeOut = 60000)
+ public void testCheckpointGcAfterMarkDeleteAbsorbsAllAcks() throws Exception {
+ ManagedLedgerConfig config = new ManagedLedgerConfig();
+ config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true);
+ config.setMaxEntriesPerLedger(2);
+ config.setMetadataMaxEntriesPerLedger(2); // frequent rollovers
+ config.setMaxUnackedRangesToPersistInMetadataStore(-1);
+ config.setThrottleMarkDelete(0);
+
+ String ledgerName = "test_checkpoint_gc";
+ ManagedLedger ledger = factory.open(ledgerName, config);
+ ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1");
+
+ List positions = new ArrayList<>();
+ for (int i = 0; i < 4; i++) {
+ positions.add(ledger.addEntry(("m-" + i).getBytes(Encoding)));
+ }
+ cursor.markDelete(positions.get(0));
+ cursor.delete(positions.get(1));
+ cursor.markDelete(positions.get(2));
+ cursor.delete(positions.get(3));
+
+ Set seenCursorLedgers = new HashSet<>();
+ seenCursorLedgers.add(cursor.getCursorLedger());
+ for (int i = 0; i < 20; i++) {
+ Position p = ledger.addEntry(("extra-" + i).getBytes(Encoding));
+ cursor.markDelete(p);
+ seenCursorLedgers.add(cursor.getCursorLedger());
+ }
+ assertThat(seenCursorLedgers.size()).isGreaterThan(1);
+
+ // At least one old cursor ledger should be GC'd (maxHold absorbed). Invoke the GC
+ // explicitly instead of relying on the incidental rollover timing: this keeps the
+ // test deterministic regardless of executor scheduling in shared-JVM test runs.
+ cursor.gcOldCursorLedgers();
+ Set seenSnapshot = Collections.unmodifiableSet(new HashSet<>(seenCursorLedgers));
+ ManagedCursorImpl cursorRef = cursor;
+ Awaitility.await().untilAsserted(() -> {
+ Set live = bkc.getLedgers();
+ long currentId = cursorRef.getCursorLedger();
+ boolean anyReclaimed = seenSnapshot.stream()
+ .anyMatch(id -> id != currentId && !live.contains(id));
+ assertThat(anyReclaimed).isTrue();
+ });
+ ledger.close();
+ }
+
+ @Test(timeOut = 60000)
+ public void testCheckpointGcRetriesAfterLedgerDeleteFailure() throws Exception {
+ ManagedLedgerConfig config = new ManagedLedgerConfig();
+ config.setPersistentUnackedRangesWithPerLedgerEntryEnabled(true);
+ config.setMaxEntriesPerLedger(2);
+ config.setMetadataMaxEntriesPerLedger(2); // frequent rollovers -> frequent GC attempts
+ config.setMaxUnackedRangesToPersistInMetadataStore(-1);
+ config.setThrottleMarkDelete(0);
+
+ String ledgerName = "test_checkpoint_gc_retry_on_delete_failure";
+ ManagedLedger ledger = factory.open(ledgerName, config);
+ ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor("c1");
+
+ List positions = new ArrayList<>();
+ for (int i = 0; i < 4; i++) {
+ positions.add(ledger.addEntry(("m-" + i).getBytes(Encoding)));
+ }
+ cursor.markDelete(positions.get(0));
+ cursor.markDelete(positions.get(3)); // everything absorbed; all refs stale
+
+ // Fail BK ledger deletes until the flag is cleared, via the mock's dedicated hook
+ // (ledger deletes do not go through the metadata store in this mock).
+ AtomicBoolean failLedgerDeletes = new AtomicBoolean(true);
+ ((org.apache.bookkeeper.client.PulsarMockBookKeeper) bkc).setDeleteLedgerFailure(failLedgerDeletes::get);
+
+ Set seenBeforeFailure = new HashSet<>();
+ for (int i = 0; i < 6; i++) {
+ Position p = ledger.addEntry(("failing-" + i).getBytes(Encoding));
+ cursor.markDelete(p);
+ seenBeforeFailure.add(cursor.getCursorLedger());
+ }
+ Set failingSnapshot = Collections.unmodifiableSet(new HashSet<>(seenBeforeFailure));
+ ManagedCursorImpl cursorDuringFailure = cursor;
+
+ // While deletes fail, old cursor ledgers must survive and GC must not lose its
+ // bookkeeping (they stay tracked for retry on the next rollover).
+ Awaitility.await().untilAsserted(() -> {
+ Set