diff --git a/src/it/README.adoc b/src/it/README.adoc index d37cbff..ce0fb9d 100644 --- a/src/it/README.adoc +++ b/src/it/README.adoc @@ -52,7 +52,9 @@ module net.openhft.hashing { } ---- -Internal packages (like sun.nio.ch) are not exported and cannot be accessed by consuming modules. +No implementation packages from this library are exported. +The direct-buffer runtime test supplies `--add-exports java.base/sun.nio.ch=net.openhft.hashing` because the performance-sensitive implementation calls `DirectBuffer.address()` directly. +For class-path use, the corresponding target is `ALL-UNNAMED`. == Multi-Release JAR @@ -65,3 +67,9 @@ The library is built as a multi-release JAR with module-info.class in META-INF/v All integration tests use JUnit 4 (same as the main project tests) and do not print to System.out. Test results are reported through Maven Surefire plugin output only. + +The library unit tests also check direct-buffer ranges independently of the active cursor window. +`LongHashFunctionTest` and `LongTupleHashFunctionTest` put distinct guard bytes around the payload, +then request only the payload while position and limit cover the complete buffer. They verify the +hash and unchanged cursor, then restore the payload window before the whole-buffer byte-order check. +Replacing the direct-buffer range with `position()` and `remaining()` must fail these assertions. diff --git a/src/it/module-test/pom.xml b/src/it/module-test/pom.xml index 3a90660..b282c10 100644 --- a/src/it/module-test/pom.xml +++ b/src/it/module-test/pom.xml @@ -36,6 +36,9 @@ maven-surefire-plugin 3.5.4 + + --add-exports java.base/sun.nio.ch=net.openhft.hashing + diff --git a/src/it/module-test/src/test/java/net/openhft/it/module/ModuleTest.java b/src/it/module-test/src/test/java/net/openhft/it/module/ModuleTest.java index ed27989..52bca90 100644 --- a/src/it/module-test/src/test/java/net/openhft/it/module/ModuleTest.java +++ b/src/it/module-test/src/test/java/net/openhft/it/module/ModuleTest.java @@ -4,9 +4,12 @@ package net.openhft.it.module; import net.openhft.hashing.LongHashFunction; +import net.openhft.hashing.LongTupleHashFunction; import org.junit.Test; import java.lang.module.ModuleDescriptor; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; import java.util.Set; @@ -64,4 +67,45 @@ public void testExplicitDescriptorContract() { assertFalse("jdk.unsupported must be available at runtime", unsupported.modifiers().contains(ModuleDescriptor.Requires.Modifier.STATIC)); } + + @Test + public void testPublicRuntimePathsWithRequiredDirectBufferExport() { + LongHashFunction[] functions = { + LongHashFunction.city_1_1(), + LongHashFunction.farmNa(), + LongHashFunction.farmUo(), + LongHashFunction.murmur_3(), + LongHashFunction.xx(), + LongHashFunction.xx3(), + LongHashFunction.xx128low(), + LongHashFunction.wy_3(), + LongHashFunction.metro() + }; + + for (LongHashFunction function : functions) { + long expected = function.hashBytes(TEST_BYTES); + assertEquals(expected, function.hashBytes(ByteBuffer.wrap(TEST_BYTES))); + + ByteBuffer direct = ByteBuffer.allocateDirect(TEST_BYTES.length); + direct.put(TEST_BYTES).flip(); + assertEquals(expected, function.hashBytes(direct)); + assertEquals("Direct buffer position should be unchanged", 0, direct.position()); + assertEquals("Direct buffer limit should be unchanged", TEST_BYTES.length, direct.limit()); + + assertEquals(function.hashChars(TEST_DATA), + function.hashChars(new StringBuilder(TEST_DATA))); + int[] integers = {1, 2, 3, 4}; + ByteBuffer integerBytes = ByteBuffer.allocate(integers.length * Integer.BYTES) + .order(ByteOrder.nativeOrder()); + for (int value : integers) + integerBytes.putInt(value); + // Primitive-array hashing uses the bytes as laid out in native memory. + assertEquals(function.hashBytes(integerBytes.array()), function.hashInts(integers)); + } + + LongTupleHashFunction tuple = LongTupleHashFunction.xx128(); + ByteBuffer direct = ByteBuffer.allocateDirect(TEST_BYTES.length); + direct.put(TEST_BYTES).flip(); + assertArrayEquals(tuple.hashBytes(TEST_BYTES), tuple.hashBytes(direct)); + } } diff --git a/src/main/docs/architecture-overview.adoc b/src/main/docs/architecture-overview.adoc index 44dce67..a9bfd12 100644 --- a/src/main/docs/architecture-overview.adoc +++ b/src/main/docs/architecture-overview.adoc @@ -36,7 +36,9 @@ It currently delivers 128-bit MurmurHash3 and XXH3 outputs and mirrors the singl * `net.openhft.hashing.Util.VALID_STRING_HASH` selects the correct `StringHash` strategy at JVM initialisation time by inspecting `java.vm.name` and `java.version`, covering HotSpot, OpenJ9, Zing, and unknown VMs (`Util.java:29-63`). * `ModernHotSpotStringHash`, `ModernCompactStringHash`, and `HotSpotPrior7u6StringHash` encode the memory layout differences between pre-compact, compact-string, and legacy HotSpot builds. When the VM cannot be recognised, `UnknownJvmStringHash` provides a defensive fallback. -* Direct buffer hashing uses `sun.nio.ch.DirectBuffer` addresses pulled via `LongHashFunction.hashBytes(ByteBuffer)` and `LongHashFunction.hashMemory(long, long)`; `Util.getDirectBufferAddress` centralises the address extraction (`Util.java:65-68`). +* Direct buffer hashing calls `sun.nio.ch.DirectBuffer.address()` and passes the native address to the allocation-free `Unsafe` path (`LongHashFunction.hashByteBuffer`, `LongTupleHashFunction.hashByteBuffer`, and `Util.getDirectBufferAddress`). +The invocation is deliberately direct rather than reflective because this is a performance-sensitive path. +On Java 9 and newer, the application must export `java.base/sun.nio.ch` to the library as described in `unsafe-and-platform-notes.adoc`. === Supporting Utilities diff --git a/src/main/docs/project-requirements.adoc b/src/main/docs/project-requirements.adoc index d6d0aa7..3fb904d 100644 --- a/src/main/docs/project-requirements.adoc +++ b/src/main/docs/project-requirements.adoc @@ -55,7 +55,7 @@ The library must implement the following non-cryptographic hash algorithms: === 3.3. Compatibility (NF-O) * **NF-O-001:** **JDK Version:** The library must support JDK 8 through the current LTS (JDK 21+). -* **NF-O-002:** **Module System:** The library must function in Java 9+ environments. Tests and runtime require `--add-opens java.base/jdk.internal.misc=ALL-UNNAMED --add-opens java.base/sun.nio.ch=ALL-UNNAMED` for `sun.misc.Unsafe` and `sun.nio.ch.DirectBuffer` access (see `unsafe-and-platform-notes.adoc`). +* **NF-O-002:** **Module System:** The library must function in Java 9+ environments as explicit module `net.openhft.hashing`. The descriptor declares the runtime dependency on `jdk.unsupported`. Direct-buffer hashing additionally requires the application to export `java.base/sun.nio.ch` to `net.openhft.hashing`; this preserves the direct, allocation-free `DirectBuffer.address()` call (see `unsafe-and-platform-notes.adoc`). * **NF-O-003:** **VM Support:** The library must support major JVMs including HotSpot, OpenJDK, and OpenJ9/IBM J9. == 4. Interface Constraints diff --git a/src/main/docs/unsafe-and-platform-notes.adoc b/src/main/docs/unsafe-and-platform-notes.adoc index 492a6eb..c427dd1 100644 --- a/src/main/docs/unsafe-and-platform-notes.adoc +++ b/src/main/docs/unsafe-and-platform-notes.adoc @@ -9,20 +9,29 @@ Chronicle Software * `UnsafeAccess` reflects on `sun.misc.Unsafe.theUnsafe` to obtain the singleton and uses it for raw array access, field offsets, and direct memory loads (`UnsafeAccess.java:40-118`). This keeps hashing allocation-free but depends on the `jdk.unsupported` module. -* Direct buffer handling casts to `sun.nio.ch.DirectBuffer` to retrieve native addresses for `hashBytes(ByteBuffer)` and `hashMemory` (`Util.java:65-68`, `LongHashFunction.java:430-470`). -The code path assumes the buffer is direct; heap buffers are copied through array access instead. +* Direct buffer handling casts to `sun.nio.ch.DirectBuffer` and calls `address()` directly before hashing the native memory. +This preserves the established zero-allocation fast path and avoids reflective invocation in a performance-sensitive operation. +There is deliberately no reflective fallback when access to `DirectBuffer` is unavailable. * String hashing inspects `java.lang.String.value` and, on compact-string VMs, treats the backing `byte[]` as Latin-1 using `CompactLatin1CharSequenceAccess` (`ModernCompactStringHash.java:10-62`). Older HotSpot builds and other JVMs fall back to UTF-16 behaviour through `ModernHotSpotStringHash` or `HotSpotPrior7u6StringHash`. === JVM Configuration -* On Java 9 and newer, strong encapsulation may block reflective access to the JDK internals above. -Add the following opens/exports when running with a strict module layer: -** `--add-opens java.base/sun.misc=ALL-UNNAMED` (access to `Unsafe.theUnsafe`). -** `--add-exports java.base/sun.nio.ch=ALL-UNNAMED` (casts to `DirectBuffer`). -** `--add-opens java.base/java.lang=ALL-UNNAMED` (reading `String.value` for compact strings). -* The library does not automatically request these permissions. -Applications embedding Zero-Allocation Hashing must configure the JVM command line or module descriptors accordingly. +* The explicit module descriptor declares `requires jdk.unsupported` for `sun.misc.Unsafe`. +* On Java 9 and newer, direct-buffer hashing also requires one of the following runtime options: +** Named-module use: `--add-exports java.base/sun.nio.ch=net.openhft.hashing` +** Class-path use: `--add-exports java.base/sun.nio.ch=ALL-UNNAMED` +The target differs because class-path code belongs to the unnamed module. +The module descriptor cannot request that another module export one of its packages, so the application launcher must supply this option. +* Java 24 and 25 emit terminal-deprecation warnings for some `sun.misc.Unsafe` memory-access methods. +The Java 25 default is `--sun-misc-unsafe-memory-access=warn`, which permits execution and emits at most one warning per JVM. +The first warning from this library normally identifies `UnsafeAccess.arrayBaseOffset`, but suppressing or replacing that one call would not resolve the underlying dependency: hashing also uses `getByte`, `getShort`, `getInt`, and `getLong` throughout its hot paths. +* Applications that have explicitly accepted this temporary compatibility risk can suppress the warning on Java 24 and 25 with `--sun-misc-unsafe-memory-access=allow`. +This is an application-wide launcher policy, not a library setting, and does not mitigate the future-removal risk. +Use `debug` instead when locating every call; `deny` makes the current implementation fail during `UnsafeAccess` initialisation. +* Removing the warning properly requires replacing all affected on-heap and off-heap memory operations with supported APIs, principally `VarHandle` and the Foreign Function and Memory API. +That work requires version-specific implementations and performance validation, and is separate from JPMS packaging. +See https://openjdk.org/jeps/498[JEP 498] and https://openjdk.org/jeps/471[JEP 471]. === Platform Behaviour diff --git a/src/main/java/sun/nio/ch/DirectBuffer.java b/src/main/java/sun/nio/ch/DirectBuffer.java index 01b65e5..76b3a4e 100644 --- a/src/main/java/sun/nio/ch/DirectBuffer.java +++ b/src/main/java/sun/nio/ch/DirectBuffer.java @@ -4,17 +4,16 @@ package sun.nio.ch; /** - * Stub for JDK internal ckass sun.nio.ch.DirectBuffer. + * Stub for JDK internal class sun.nio.ch.DirectBuffer. *

- * - When crossing compiling for Java SE 7 and 8, this stub class bypasses compiler sun-api + * - When cross compiling for Java SE 7 and 8, this stub class bypasses compiler sun-api * warnings. - * - When crossing compiling for Java SE 9+, the package 'sun.nio.ch' is not exported from - * 'java.base' module. This stub class helps to access the class in compile-time without - * '--add-export' arguments and bypasses sun-api warnings. - * - Only used methods are exported. - * - In test and production runtime, the real class is loaded from boot classpath. + * - When cross compiling for Java SE 9+, the package 'sun.nio.ch' is not exported from + * 'java.base'. This stub class helps to access the class at compile time without + * '--add-exports' arguments and bypasses sun-api warnings. + * - Only used methods are declared. + * - In test and production runtime, the real class is loaded from the boot class path. */ - public interface DirectBuffer { - public long address(); + long address(); } diff --git a/src/test/java/net/openhft/hashing/AutomaticModuleNameIT.java b/src/test/java/net/openhft/hashing/AutomaticModuleNameIT.java index 882b808..ad4acd2 100644 --- a/src/test/java/net/openhft/hashing/AutomaticModuleNameIT.java +++ b/src/test/java/net/openhft/hashing/AutomaticModuleNameIT.java @@ -15,6 +15,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; public class AutomaticModuleNameIT { @@ -54,4 +55,16 @@ public void java8BuildProducesJava9ModuleDescriptor() throws IOException { } } } + + @Test + public void packagedJarDoesNotContainJdkCompilationStub() throws IOException { + final String packagedJarPath = System.getProperty("packaged.jar"); + assertNotNull("packaged.jar system property", packagedJarPath); + + try (JarFile jarFile = new JarFile(new File(packagedJarPath))) { + assertNull("The JDK supplies sun.nio.ch.DirectBuffer at runtime; the compile-time stub " + + "must not be packaged", + jarFile.getJarEntry("sun/nio/ch/DirectBuffer.class")); + } + } } diff --git a/src/test/java/net/openhft/hashing/LongHashFunctionTest.java b/src/test/java/net/openhft/hashing/LongHashFunctionTest.java index fa4380a..dc3277f 100644 --- a/src/test/java/net/openhft/hashing/LongHashFunctionTest.java +++ b/src/test/java/net/openhft/hashing/LongHashFunctionTest.java @@ -152,11 +152,34 @@ private static void testByteBuffers(LongHashFunction f, long eh, int len, ByteBu bb2.put(bb); assertEquals("byte buffer little endian off len", eh, f.hashBytes(bb2, 1, len)); + ByteBuffer direct = ByteBuffer.allocateDirect(len + 2).order(LITTLE_ENDIAN); + direct.put(0, (byte) 0x6D); + direct.put(len + 1, (byte) 0xB7); + ((Buffer)direct).position(1); + ByteBuffer directSource = bb.duplicate(); + ((Buffer)directSource).clear(); + direct.put(directSource); + ((Buffer)direct).position(1); + ((Buffer)direct).limit(len + 1); + assertEquals("direct byte buffer", eh, f.hashBytes(direct)); + assertEquals("direct byte buffer position unchanged", 1, direct.position()); + assertEquals("direct byte buffer limit unchanged", len + 1, direct.limit()); + + // The explicit range excludes guards that are inside the active window. + ((Buffer)direct).clear(); + assertEquals("direct byte buffer off len", eh, f.hashBytes(direct, 1, len)); + assertEquals("direct range position unchanged", 0, direct.position()); + assertEquals("direct range limit unchanged", len + 2, direct.limit()); + ((Buffer)direct).position(1); + ((Buffer)direct).limit(len + 1); + ((Buffer)bb.order(BIG_ENDIAN)).clear(); assertEquals("byte buffer big endian", eh, f.hashBytes(bb)); bb2.order(BIG_ENDIAN); assertEquals("byte buffer big endian off len", eh, f.hashBytes(bb2, 1, len)); + direct.order(BIG_ENDIAN); + assertEquals("direct byte buffer big endian", eh, f.hashBytes(direct)); ((Buffer)bb.order(nativeOrder())).clear(); } diff --git a/src/test/java/net/openhft/hashing/LongTupleHashFunctionTest.java b/src/test/java/net/openhft/hashing/LongTupleHashFunctionTest.java index 9781aa2..78c2b36 100644 --- a/src/test/java/net/openhft/hashing/LongTupleHashFunctionTest.java +++ b/src/test/java/net/openhft/hashing/LongTupleHashFunctionTest.java @@ -204,11 +204,34 @@ private static void testByteBuffers(LongTupleHashFunction f, long[] eh, int len, bb2.put(bb); assertArrayEquals("byte buffer little endian off len", eh, f.hashBytes(bb2, 1, len)); + ByteBuffer direct = ByteBuffer.allocateDirect(len + 2).order(LITTLE_ENDIAN); + direct.put(0, (byte) 0x6D); + direct.put(len + 1, (byte) 0xB7); + ((Buffer)direct).position(1); + ByteBuffer directSource = bb.duplicate(); + ((Buffer)directSource).clear(); + direct.put(directSource); + ((Buffer)direct).position(1); + ((Buffer)direct).limit(len + 1); + assertArrayEquals("direct byte buffer", eh, f.hashBytes(direct)); + assertEquals("direct byte buffer position unchanged", 1, direct.position()); + assertEquals("direct byte buffer limit unchanged", len + 1, direct.limit()); + + // The explicit range excludes guards that are inside the active window. + ((Buffer)direct).clear(); + assertArrayEquals("direct byte buffer off len", eh, f.hashBytes(direct, 1, len)); + assertEquals("direct range position unchanged", 0, direct.position()); + assertEquals("direct range limit unchanged", len + 2, direct.limit()); + ((Buffer)direct).position(1); + ((Buffer)direct).limit(len + 1); + ((Buffer)bb.order(BIG_ENDIAN)).clear(); assertArrayEquals("byte buffer big endian", eh, f.hashBytes(bb)); bb2.order(BIG_ENDIAN); assertArrayEquals("byte buffer big endian off len", eh, f.hashBytes(bb2, 1, len)); + direct.order(BIG_ENDIAN); + assertArrayEquals("direct byte buffer big endian", eh, f.hashBytes(direct)); ((Buffer)bb.order(nativeOrder())).clear(); }