[improve][broker] Bridge JUL to Log4j2 to unify third-party library logging under the project's Log4j2 configuration

bin/bookkeeper

@@ -196,6 +196,9 @@ OPTS="-cp $BOOKIE_CLASSPATH $OPTS"
 
 # Disable ipv6 as it can cause issues
 OPTS="-Djava.net.preferIPv4Stack=true $OPTS"
+# Bridge java.util.logging (JUL) to Log4j2 so that JUL logs from third-party libraries
+# (Jersey, gRPC, Guava, etc.) are bridged into the Log4j2 configuration (conf/log4j2.yaml)
+OPTS="-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager $OPTS"
 # Required to allow sun.misc.Unsafe on JDK 24 without warnings
 # Also required for enabling unsafe memory access for Netty since 4.1.121.Final
 if [[ $JAVA_MAJOR_VERSION -ge 23 ]]; then

bin/function-localrunner

@@ -121,6 +121,9 @@ fi
 # rarely needed when trying to list many z-nodes under a
 # directory)
 OPTS="-Djava.net.preferIPv4Stack=true $OPTS -Djute.maxbuffer=10485760"
+# Bridge java.util.logging (JUL) to Log4j2 so that JUL logs from third-party libraries
+# (Jersey, gRPC, Guava, etc.) are bridged into the Log4j2 configuration (conf/log4j2.yaml)
+OPTS="-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager $OPTS"
 # Required to allow sun.misc.Unsafe on JDK 24 without warnings
 # Also required for enabling unsafe memory access for Netty since 4.1.121.Final
 if [[ $JAVA_MAJOR_VERSION -ge 23 ]]; then

bin/pulsar

@@ -267,6 +267,9 @@ OPTS="$OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF`"
 # rarely needed when trying to list many z-nodes under a
 # directory)
 OPTS="-Djava.net.preferIPv4Stack=true $OPTS -Djute.maxbuffer=10485760"
+# Bridge java.util.logging (JUL) to Log4j2 so that JUL logs from third-party libraries
+# (Jersey, gRPC, Guava, etc.) are bridged into the Log4j2 configuration (conf/log4j2.yaml)
+OPTS="-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager $OPTS"
 # Required to allow sun.misc.Unsafe on JDK 24 without warnings
 # Also required for enabling unsafe memory access for Netty since 4.1.121.Final
 if [[ $JAVA_MAJOR_VERSION -ge 23 ]]; then

bin/pulsar-admin-common.cmd

@@ -61,6 +61,9 @@ set "PULSAR_CLASSPATH=%PULSAR_CLASSPATH%;%PULSAR_LOG_CONF_DIR%"
 
 set "OPTS=%OPTS% -Dlog4j.configurationFile="%PULSAR_LOG_CONF_BASENAME%""
 set "OPTS=-Djava.net.preferIPv4Stack=true %OPTS%"
+REM Bridge java.util.logging (JUL) to Log4j2 so that JUL logs from third-party libraries
+REM (Jersey, gRPC, Guava, etc.) are bridged into the Log4j2 configuration (conf/log4j2.yaml)
+set "OPTS=-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager %OPTS%"
 
 REM Allow Netty to use reflection access
 set "OPTS=%OPTS% -Dio.netty.tryReflectionSetAccessible=true"

bin/pulsar-admin-common.sh

@@ -97,6 +97,9 @@ PULSAR_CLASSPATH="$PULSAR_JAR:$PULSAR_CLASSPATH:$PULSAR_EXTRA_CLASSPATH"
 PULSAR_CLASSPATH="`dirname $PULSAR_LOG_CONF`:$PULSAR_CLASSPATH"
 OPTS="$OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF`"
 OPTS="-Djava.net.preferIPv4Stack=true $OPTS"
+# Bridge java.util.logging (JUL) to Log4j2 so that JUL logs from third-party libraries
+# (Jersey, gRPC, Guava, etc.) are bridged into the Log4j2 configuration (conf/log4j2.yaml)
+OPTS="-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager $OPTS"
 # Required to allow sun.misc.Unsafe on JDK 24 without warnings
 # Also required for enabling unsafe memory access for Netty since 4.1.121.Final
 if [[ $JAVA_MAJOR_VERSION -ge 23 ]]; then

bin/pulsar-perf

@@ -94,6 +94,9 @@ fi
 PULSAR_CLASSPATH="$PULSAR_JAR:$PULSAR_CLASSPATH:$PULSAR_EXTRA_CLASSPATH"
 PULSAR_CLASSPATH="`dirname $PULSAR_LOG_CONF`:$PULSAR_CLASSPATH"
 OPTS="-Djava.net.preferIPv4Stack=true $OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF`"
+# Bridge java.util.logging (JUL) to Log4j2 so that JUL logs from third-party libraries
+# (Jersey, gRPC, Guava, etc.) are bridged into the Log4j2 configuration (conf/log4j2.yaml)
+OPTS="-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager $OPTS"
 # Required to allow sun.misc.Unsafe on JDK 24 without warnings
 # Also required for enabling unsafe memory access for Netty since 4.1.121.Final
 if [[ $JAVA_MAJOR_VERSION -ge 23 ]]; then

build-logic/conventions/src/main/kotlin/pulsar.java-conventions.gradle.kts

@@ -193,6 +193,8 @@ dependencies {
     "testImplementation"(catalog.findLibrary("awaitility").get())
     "testImplementation"(catalog.findLibrary("system-lambda").get())
     "testImplementation"(catalog.findLibrary("slf4j-api").get())
+    // log4j-jul is needed at test runtime to support the JUL bridge JVM argument above
+    "testRuntimeOnly"(catalog.findLibrary("log4j-jul").get())
 }
 
 // Allow overriding the JDK used for running tests via -PtestJavaVersion=17
@@ -258,6 +260,9 @@ tasks.withType<Test>().configureEach {
         "-Dpulsar.allocator.exit_on_oom=false",
         "-Dpulsar.allocator.out_of_memory_policy=FallbackToHeap",
         "-Dpulsar.test.preventExit=true",
+        // Bridge java.util.logging (JUL) to Log4j2 so that JUL logs from third-party libraries
+        // (Jersey, gRPC, Guava, etc.) are routed to Log4j2 instead of stdout
+        "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager",
         // Force IPv4 to match Pulsar's runtime scripts (bin/pulsar, bin/bookkeeper). BookKeeper's
         // BookieId validation rejects IPv6 zone identifiers (e.g. fe80::1%lo0), so on hosts where the
         // loopback interface resolves to an IPv6 link-local address (notably macOS) bookies bound to

distribution/server/build.gradle.kts

@@ -124,6 +124,7 @@ dependencies {
     distLib(libs.log4j.web)
     distLib(libs.log4j.layout.template.json)
     distLib(libs.log4j.slf4j2.impl)
+    distLib(libs.log4j.jul)
     distLib(libs.simpleclient.log4j2)
 
     // Metrics

distribution/server/src/assemble/LICENSE.bin.txt

@@ -350,6 +350,7 @@ The Apache Software License, Version 2.0
  * Log4J
     - org.apache.logging.log4j-log4j-api-2.26.1.jar
     - org.apache.logging.log4j-log4j-core-2.26.1.jar
+    - org.apache.logging.log4j-log4j-jul-2.26.1.jar
     - org.apache.logging.log4j-log4j-slf4j2-impl-2.26.1.jar
     - org.apache.logging.log4j-log4j-web-2.26.1.jar
     - org.apache.logging.log4j-log4j-layout-template-json-2.26.1.jar

distribution/shell/build.gradle.kts

@@ -55,6 +55,7 @@ dependencies {
     distLib(libs.log4j.web)
     distLib(libs.log4j.layout.template.json)
     distLib(libs.log4j.slf4j2.impl)
+    distLib(libs.log4j.jul)
     distLib(libs.simpleclient.log4j2)
     // Bouncy Castle (non-FIPS JCA provider for client-side message crypto + TLS)
     distLib(libs.bcprov.jdk18on)

distribution/shell/src/assemble/LICENSE.bin.txt

@@ -385,6 +385,7 @@ The Apache Software License, Version 2.0
  * Log4J
     - log4j-api-2.26.1.jar
     - log4j-core-2.26.1.jar
+    - log4j-jul-2.26.1.jar
     - log4j-layout-template-json-2.26.1.jar
     - log4j-slf4j2-impl-2.26.1.jar
     - log4j-web-2.26.1.jar

gradle/libs.versions.toml

@@ -215,6 +215,7 @@ log4j-core = { module = "org.apache.logging.log4j:log4j-core", version.ref = "lo
 log4j-web = { module = "org.apache.logging.log4j:log4j-web", version.ref = "log4j2" }
 log4j-layout-template-json = { module = "org.apache.logging.log4j:log4j-layout-template-json", version.ref = "log4j2" }
 log4j-slf4j2-impl = { module = "org.apache.logging.log4j:log4j-slf4j2-impl", version.ref = "log4j2" }
+log4j-jul = { module = "org.apache.logging.log4j:log4j-jul", version.ref = "log4j2" }
 # slog
 slog = { module = "io.github.merlimat.slog:slog", version.ref = "slog" }
 # Lombok

pulsar-broker/src/test/java/org/apache/pulsar/utils/JulBridgeTest.java

@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.utils;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+import java.util.Optional;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.apache.logging.log4j.core.LogEvent;
+import org.testng.annotations.Test;
+
+/**
+ * Verify that the JUL-to-Log4j2 bridge is active in the test JVM and that
+ * log events emitted via {@link java.util.logging} are routed to Log4j2.
+ */
+public class JulBridgeTest {
+
+    @Test
+    public void testJulBridgeIsActive() {
+        // The JVM should have been started with
+        // -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager
+        java.util.logging.LogManager logManager = java.util.logging.LogManager.getLogManager();
+        assertEquals(logManager.getClass().getName(),
+                "org.apache.logging.log4j.jul.LogManager",
+                "JUL bridge should be active via -Djava.util.logging.manager system property");
+    }
+
+    @Test
+    public void testJulLogsAreRoutedToLog4j2() throws Exception {
+        String loggerName = "org.apache.pulsar.test.jul.bridge";
+
+        // Attach a TestLogAppender to capture log events for this logger
+        try (TestLogAppender appender = TestLogAppender.create(Optional.of(loggerName))) {
+            // Log via java.util.logging API
+            Logger julLogger = Logger.getLogger(loggerName);
+            julLogger.severe("JUL SEVERE test message");
+            julLogger.warning("JUL WARNING test message");
+            julLogger.info("JUL INFO test message");
+
+            // Verify that the JUL log events were routed to Log4j2
+            assertTrue(appender.getEvents().stream()
+                            .anyMatch(e -> e.getMessage().getFormattedMessage().contains("JUL SEVERE test message")
+                                    && e.getLevel() == org.apache.logging.log4j.Level.ERROR),
+                    "JUL SEVERE should be routed to Log4j2 as ERROR");
+
+            assertTrue(appender.getEvents().stream()
+                            .anyMatch(e -> e.getMessage().getFormattedMessage().contains("JUL WARNING test message")
+                                    && e.getLevel() == org.apache.logging.log4j.Level.WARN),
+                    "JUL WARNING should be routed to Log4j2 as WARN");
+
+            assertTrue(appender.getEvents().stream()
+                            .anyMatch(e -> e.getMessage().getFormattedMessage().contains("JUL INFO test message")
+                                    && e.getLevel() == org.apache.logging.log4j.Level.INFO),
+                    "JUL INFO should be routed to Log4j2 as INFO");
+        }
+    }
+
+    @Test
+    public void testJulExceptionIsPreserved() throws Exception {
+        String loggerName = "org.apache.pulsar.test.jul.exception";
+
+        try (TestLogAppender appender = TestLogAppender.create(Optional.of(loggerName))) {
+            Logger julLogger = Logger.getLogger(loggerName);
+            RuntimeException testException = new RuntimeException("test exception from JUL");
+            julLogger.log(Level.SEVERE, "Error occurred", testException);
+
+            // Verify that the exception is preserved in the Log4j2 event
+            Optional<LogEvent> event = appender.getEvents().stream()
+                    .filter(e -> e.getMessage().getFormattedMessage().contains("Error occurred"))
+                    .findFirst();
+            assertTrue(event.isPresent(), "Should find the log event");
+            assertTrue(event.get().getThrown() instanceof RuntimeException,
+                    "Exception should be preserved");
+            assertEquals(event.get().getThrown().getMessage(), "test exception from JUL");
+        }
+    }
+}

pulsar-functions/runtime-all/build.gradle.kts

@@ -36,6 +36,8 @@ dependencies {
     implementation(libs.log4j.slf4j2.impl)
     implementation(libs.log4j.api)
     implementation(libs.log4j.core)
+    // log4j-jul is needed to support the JUL-to-Log4j2 bridge in function instance JVMs
+    implementation(libs.log4j.jul)
 }
 
 // Build a fat JAR as java-instance.jar using the Shadow plugin.

pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java

@@ -359,6 +359,9 @@ public static List<String> getCmd(InstanceConfig instanceConfig,
                 args.add(String.format("-D%s=%s", FUNCTIONS_INSTANCE_CLASSPATH, systemFunctionInstanceClasspath));
             }
             args.add("-Dlog4j.configurationFile=" + logConfigFile);
+            // Bridge java.util.logging (JUL) to Log4j2 so that JUL logs from third-party libraries
+            // (Jersey, gRPC, Guava, etc.) are bridged into the Log4j2 configuration
+            args.add("-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager");
             // Use a single LoggerContext for the function process. The default
             // ClassLoaderContextSelector creates separate contexts per classloader,
             // which causes the LogAppender (added from the instance classloader) to

pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java

@@ -485,14 +485,14 @@ private void verifyJavaInstance(InstanceConfig config, String depsDir, boolean s
         if (null != depsDir) {
             extraDepsEnv = " -Dpulsar.functions.extra.dependencies.dir=" + depsDir;
             classpath = classpath + ":" + depsDir + "/*";
-            totalArgs = 53;
-            portArg = 40;
-            metricsPortArg = 42;
+            totalArgs = 54;
+            portArg = 41;
+            metricsPortArg = 43;
         } else {
             extraDepsEnv = "";
-            portArg = 39;
-            metricsPortArg = 41;
-            totalArgs = 52;
+            portArg = 40;
+            metricsPortArg = 42;
+            totalArgs = 53;
         }
         if (secretsAttached) {
             totalArgs += 4;
@@ -520,6 +520,7 @@ private void verifyJavaInstance(InstanceConfig config, String depsDir, boolean s
                 + extraDepsEnv
                 + " -Dpulsar.functions.instance.classpath=/pulsar/lib/*"
                 + " -Dlog4j.configurationFile=kubernetes_instance_log4j2.xml"
+                + " -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"
                 + " -Dlog4j2.contextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector "
                 + "-Dpulsar.function.log.dir=" + logDirectory + "/"
                 + FunctionCommon.getFullyQualifiedName(config.getFunctionDetails())

pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeTest.java

@@ -299,21 +299,21 @@ private void verifyJavaInstance(InstanceConfig config, Path depsDir, String webS
         String extraDepsEnv;
         int portArg;
         int metricsPortArg;
-        int totalArgCount = 55;
+        int totalArgCount = 56;
         if (webServiceUrl != null && config.isExposePulsarAdminClientEnabled()) {
             totalArgCount += 3;
         }
         if (null != depsDir) {
             assertEquals(args.size(), totalArgCount);
             extraDepsEnv = " -Dpulsar.functions.extra.dependencies.dir=" + depsDir;
             classpath = classpath + ":" + depsDir + "/*";
-            portArg = 38;
-            metricsPortArg = 40;
+            portArg = 39;
+            metricsPortArg = 41;
         } else {
             assertEquals(args.size(), totalArgCount - 1);
             extraDepsEnv = "";
-            portArg = 37;
-            metricsPortArg = 39;
+            portArg = 38;
+            metricsPortArg = 40;
         }
         if (webServiceUrl != null && config.isExposePulsarAdminClientEnabled()) {
             portArg += 3;
@@ -327,6 +327,7 @@ private void verifyJavaInstance(InstanceConfig config, Path depsDir, String webS
                 + extraDepsEnv
                 + " -Dpulsar.functions.instance.classpath=/pulsar/lib/*"
                 + " -Dlog4j.configurationFile=java_instance_log4j2.xml"
+                + " -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"
                 + " -Dlog4j2.contextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector "
                 + "-Dpulsar.function.log.dir=" + logDirectory + "/functions/"
                 + FunctionCommon.getFullyQualifiedName(config.getFunctionDetails())