From bfa90b24bacdec1ddb8a9f6b7231f902b93c5e7a Mon Sep 17 00:00:00 2001 From: Tyler Williams Date: Mon, 26 Jun 2023 16:41:19 -0700 Subject: [PATCH] Add BLAKE3 hasher to vfs --- .../devtools/build/lib/util/Fingerprint.java | 49 +++-- .../com/google/devtools/build/lib/vfs/BUILD | 2 + .../build/lib/vfs/Blake3HashFunction.java | 65 +++++++ .../devtools/build/lib/vfs/Blake3Hasher.java | 179 ++++++++++++++++++ .../devtools/build/lib/vfs/Blake3JNI.java | 34 ++++ .../build/lib/vfs/DigestHashFunction.java | 59 +----- src/main/native/BUILD | 15 ++ src/main/native/blake3_jni.cc | 80 ++++++++ .../build/lib/vfs/Blake3HasherTest.java | 48 +++++ .../lib/vfs/DigestHashFunctionsTest.java | 61 ------ third_party/blake3/blake3.BUILD | 5 +- 11 files changed, 457 insertions(+), 140 deletions(-) create mode 100644 src/main/java/com/google/devtools/build/lib/vfs/Blake3HashFunction.java create mode 100644 src/main/java/com/google/devtools/build/lib/vfs/Blake3Hasher.java create mode 100644 src/main/java/com/google/devtools/build/lib/vfs/Blake3JNI.java create mode 100644 src/main/native/blake3_jni.cc create mode 100644 src/test/java/com/google/devtools/build/lib/vfs/Blake3HasherTest.java delete mode 100644 src/test/java/com/google/devtools/build/lib/vfs/DigestHashFunctionsTest.java diff --git a/src/main/java/com/google/devtools/build/lib/util/Fingerprint.java b/src/main/java/com/google/devtools/build/lib/util/Fingerprint.java index 2b713adff59075..b146b3bbe9be24 100644 --- a/src/main/java/com/google/devtools/build/lib/util/Fingerprint.java +++ b/src/main/java/com/google/devtools/build/lib/util/Fingerprint.java @@ -14,9 +14,10 @@ package com.google.devtools.build.lib.util; -import static java.nio.charset.StandardCharsets.UTF_8; -import com.google.common.io.ByteStreams; +import com.google.common.hash.Funnels; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hasher; import com.google.devtools.build.lib.vfs.DigestHashFunction; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; @@ -24,16 +25,14 @@ import com.google.protobuf.ByteString; import com.google.protobuf.CodedOutputStream; import java.io.IOException; -import java.security.DigestException; -import java.security.DigestOutputStream; -import java.security.MessageDigest; +import java.nio.charset.StandardCharsets; import java.util.Collection; import java.util.Map; import java.util.UUID; import javax.annotation.Nullable; /** - * Simplified wrapper for using {@link MessageDigest} to generate fingerprints. + * Simplified wrapper for using {@link HashFunction} to generate fingerprints. * *

A fingerprint is a cryptographic hash of a message that encodes the representation of an * object. Two objects of the same type have the same fingerprint if and only if they are equal. @@ -64,18 +63,14 @@ public final class Fingerprint { // Make novel use of a CodedOutputStream, which is good at efficiently serializing data. By // flushing at the end of each digest we can continue to use the stream. - private final CodedOutputStream codedOut; - private final MessageDigest messageDigest; + private final HashFunction hashFunction; + private Hasher hasher; + private CodedOutputStream codedOut; /** Creates and initializes a new instance. */ public Fingerprint(DigestHashFunction digestFunction) { - messageDigest = digestFunction.cloneOrCreateMessageDigest(); - // This is a lot of indirection, but CodedOutputStream does a reasonable job of converting - // strings to bytes without creating a whole bunch of garbage, which pays off. - codedOut = - CodedOutputStream.newInstance( - new DigestOutputStream(ByteStreams.nullOutputStream(), messageDigest), - /*bufferSize=*/ 1024); + hashFunction = digestFunction.getHashFunction(); + reset(); } public Fingerprint() { @@ -84,6 +79,14 @@ public Fingerprint() { this(DigestHashFunction.SHA256); } + private void reset() { + hasher = hashFunction.newHasher(); + // This is a lot of indirection, but CodedOutputStream does a reasonable job of converting + // strings to bytes without creating a whole bunch of garbage, which pays off. + codedOut = + CodedOutputStream.newInstance(Funnels.asOutputStream(hasher), /* bufferSize= */ 1024); + } + /** * Completes the hash computation by doing final operations and resets the underlying state, * allowing this instance to be used again. @@ -97,7 +100,9 @@ public byte[] digestAndReset() { } catch (IOException e) { throw new IllegalStateException("failed to flush", e); } - return messageDigest.digest(); + byte[] digest = hasher.hash().asBytes(); + reset(); + return digest; } /** @@ -112,12 +117,12 @@ public byte[] digestAndReset() { public void digestAndReset(byte[] buf, int offset, int len) { try { codedOut.flush(); - messageDigest.digest(buf, offset, len); } catch (IOException e) { throw new IllegalStateException("failed to flush", e); - } catch (DigestException e) { - throw new IllegalStateException("failed to digest", e); } + + hasher.hash().writeBytesTo(buf, offset, len); + reset(); } /** Same as {@link #digestAndReset()}, except returns the digest in hex string form. */ @@ -343,7 +348,9 @@ public static String getHexDigest(String input) { // use the value from DigestHashFunction.getDefault(). However, this gets called during class // loading in a few places, before setDefault() has been called, so these call-sites should be // removed before this can be done safely. - return hexDigest( - DigestHashFunction.SHA256.cloneOrCreateMessageDigest().digest(input.getBytes(UTF_8))); + return DigestHashFunction.SHA256 + .getHashFunction() + .hashString(input, StandardCharsets.UTF_8) + .toString(); } } diff --git a/src/main/java/com/google/devtools/build/lib/vfs/BUILD b/src/main/java/com/google/devtools/build/lib/vfs/BUILD index f80997f0305041..22dd4272cafd36 100644 --- a/src/main/java/com/google/devtools/build/lib/vfs/BUILD +++ b/src/main/java/com/google/devtools/build/lib/vfs/BUILD @@ -66,6 +66,7 @@ java_library( ":pathfragment", "//src/main/java/com/google/devtools/build/lib/clock", "//src/main/java/com/google/devtools/build/lib/concurrent", + "//src/main/java/com/google/devtools/build/lib/jni", "//src/main/java/com/google/devtools/build/lib/io:file_symlink_exception", "//src/main/java/com/google/devtools/build/lib/profiler", "//src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec", @@ -77,6 +78,7 @@ java_library( "//third_party:guava", "//third_party:jsr305", "//third_party/protobuf:protobuf_java", + "@maven//:com_google_errorprone_error_prone_annotations", ], ) diff --git a/src/main/java/com/google/devtools/build/lib/vfs/Blake3HashFunction.java b/src/main/java/com/google/devtools/build/lib/vfs/Blake3HashFunction.java new file mode 100644 index 00000000000000..d9dbe9b421bf47 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/Blake3HashFunction.java @@ -0,0 +1,65 @@ +package com.google.devtools.build.lib.vfs; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkPositionIndexes; + +import com.google.common.hash.Funnel; +import com.google.common.hash.HashCode; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hasher; +import com.google.errorprone.annotations.Immutable; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; + +@Immutable +public final class Blake3HashFunction implements HashFunction { + public int bits() { + return 256; + } + + public Hasher newHasher() { + return new Blake3Hasher(); + } + + /* The following methods implement the {Hasher} interface. */ + + public HashCode hashObject(T instance, Funnel funnel) { + return newHasher().putObject(instance, funnel).hash(); + } + + public HashCode hashUnencodedChars(CharSequence input) { + int len = input.length(); + return newHasher(len * 2).putUnencodedChars(input).hash(); + } + + public HashCode hashString(CharSequence input, Charset charset) { + return newHasher().putString(input, charset).hash(); + } + + public HashCode hashInt(int input) { + return newHasher(4).putInt(input).hash(); + } + + public HashCode hashLong(long input) { + return newHasher(8).putLong(input).hash(); + } + + public HashCode hashBytes(byte[] input) { + return hashBytes(input, 0, input.length); + } + + public HashCode hashBytes(byte[] input, int off, int len) { + checkPositionIndexes(off, off + len, input.length); + return newHasher(len).putBytes(input, off, len).hash(); + } + + public HashCode hashBytes(ByteBuffer input) { + return newHasher(input.remaining()).putBytes(input).hash(); + } + + public Hasher newHasher(int expectedInputSize) { + checkArgument( + expectedInputSize >= 0, "expectedInputSize must be >= 0 but was %s", expectedInputSize); + return newHasher(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/vfs/Blake3Hasher.java b/src/main/java/com/google/devtools/build/lib/vfs/Blake3Hasher.java new file mode 100644 index 00000000000000..afc6b2f4084697 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/Blake3Hasher.java @@ -0,0 +1,179 @@ +package com.google.devtools.build.lib.vfs; + +import static com.google.common.base.Preconditions.checkState; + +import com.google.common.hash.Funnel; +import com.google.common.hash.HashCode; +import com.google.common.hash.Hasher; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; + +final class Blake3Hasher implements Hasher { + // These constants match the native definitions in: + // https://github.com/BLAKE3-team/BLAKE3/blob/master/c/blake3.h + public static final int KEY_LEN = 32; + public static final int OUT_LEN = 32; + + // To reduce the number of calls made via JNI, buffer up to this many bytes. + // If a call to "hash()" is made and less than this much data has been + // written, a single JNI call will be made that initializes, hashes, and + // cleans up the hasher, rather than making separate calls for each operation. + public static final int ONESHOT_THRESHOLD = 8 * 1024; + private static ThreadLocal threadLocalBuffer = new ThreadLocal(); + private ByteBuffer buffer = null; + + private long hasher = -1; + private boolean isDone; + + public Blake3Hasher() { + isDone = false; + + buffer = threadLocalBuffer.get(); + if (buffer == null) { + buffer = ByteBuffer.allocate(ONESHOT_THRESHOLD); + threadLocalBuffer.set(buffer); + } + } + + private void flush() { + if (hasher == -1) { + hasher = Blake3JNI.allocate_and_initialize_hasher(); + } + + if (buffer.position() > 0) { + Blake3JNI.blake3_hasher_update(hasher, buffer.array(), buffer.position()); + buffer.clear(); + } + } + + private void update(byte[] data, int offset, int length) { + while (length > 0) { + int numToCopy = Math.min(length, buffer.remaining()); + buffer.put(data, offset, numToCopy); + length -= numToCopy; + offset += numToCopy; + + if (buffer.remaining() == 0) { + flush(); + } + } + } + + private void update(byte[] data) { + update(data, 0, data.length); + } + + private byte[] getOutput(int outputLength) throws IllegalArgumentException { + byte[] retByteArray = new byte[outputLength]; + + checkState(!isDone); + isDone = true; + + if (hasher == -1) { + // If no flush has happened yet; oneshot this. + Blake3JNI.oneshot(buffer.array(), buffer.position(), retByteArray, outputLength); + buffer.clear(); + } else { + flush(); + Blake3JNI.blake3_hasher_finalize_and_close(hasher, retByteArray, outputLength); + hasher = -1; + } + return retByteArray; + } + + /* The following methods implement the {Hasher} interface. */ + + @CanIgnoreReturnValue + public Hasher putBytes(ByteBuffer b) { + buffer = b; + return this; + } + + @CanIgnoreReturnValue + public Hasher putBytes(byte[] bytes, int off, int len) { + update(bytes, off, len); + return this; + } + + @CanIgnoreReturnValue + public Hasher putBytes(byte[] bytes) { + update(bytes, 0, bytes.length); + return this; + } + + @CanIgnoreReturnValue + public Hasher putByte(byte b) { + update(new byte[] {b}); + return this; + } + + public HashCode hash() { + return HashCode.fromBytes(getOutput(OUT_LEN)); + } + + @CanIgnoreReturnValue + public final Hasher putBoolean(boolean b) { + return putByte(b ? (byte) 1 : (byte) 0); + } + + @CanIgnoreReturnValue + public final Hasher putDouble(double d) { + return putLong(Double.doubleToRawLongBits(d)); + } + + @CanIgnoreReturnValue + public final Hasher putFloat(float f) { + return putInt(Float.floatToRawIntBits(f)); + } + + @CanIgnoreReturnValue + public Hasher putUnencodedChars(CharSequence charSequence) { + for (int i = 0, len = charSequence.length(); i < len; i++) { + putChar(charSequence.charAt(i)); + } + return this; + } + + @CanIgnoreReturnValue + public Hasher putString(CharSequence charSequence, Charset charset) { + return putBytes(charSequence.toString().getBytes(charset)); + } + + @CanIgnoreReturnValue + public Hasher putShort(short s) { + putByte((byte) s); + putByte((byte) (s >>> 8)); + return this; + } + + @CanIgnoreReturnValue + public Hasher putInt(int i) { + putByte((byte) i); + putByte((byte) (i >>> 8)); + putByte((byte) (i >>> 16)); + putByte((byte) (i >>> 24)); + return this; + } + + @CanIgnoreReturnValue + public Hasher putLong(long l) { + for (int i = 0; i < 64; i += 8) { + putByte((byte) (l >>> i)); + } + return this; + } + + @CanIgnoreReturnValue + public Hasher putChar(char c) { + putByte((byte) c); + putByte((byte) (c >>> 8)); + return this; + } + + @CanIgnoreReturnValue + public Hasher putObject(T instance, Funnel funnel) { + funnel.funnel(instance, this); + return this; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/vfs/Blake3JNI.java b/src/main/java/com/google/devtools/build/lib/vfs/Blake3JNI.java new file mode 100644 index 00000000000000..5fe12555cd0054 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/Blake3JNI.java @@ -0,0 +1,34 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.vfs; + +import com.google.devtools.build.lib.jni.JniLoader; + +final class Blake3JNI { + private Blake3JNI() {} + + static { + JniLoader.loadJni(); + } + + public static final native long allocate_and_initialize_hasher(); + + public static final native void blake3_hasher_update(long self, byte[] input, int input_len); + + public static final native void blake3_hasher_finalize_and_close( + long self, byte[] out, int out_len); + + public static final native void oneshot(byte[] input, int input_len, byte[] out, int out_len); +} diff --git a/src/main/java/com/google/devtools/build/lib/vfs/DigestHashFunction.java b/src/main/java/com/google/devtools/build/lib/vfs/DigestHashFunction.java index e279d7b95b7c39..b1e5ede4c8d6d5 100644 --- a/src/main/java/com/google/devtools/build/lib/vfs/DigestHashFunction.java +++ b/src/main/java/com/google/devtools/build/lib/vfs/DigestHashFunction.java @@ -23,18 +23,10 @@ import com.google.devtools.build.lib.vfs.DigestHashFunction.DigestLength.DigestLengthImpl; import com.google.devtools.common.options.Converter; import com.google.devtools.common.options.OptionsParsingException; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.util.HashMap; import java.util.Map.Entry; -/** - * Type of hash function to use for digesting files. - * - *

This tracks parallel {@link java.security.MessageDigest} and {@link HashFunction} interfaces - * for each provided hash, as Bazel uses both - MessageDigest where performance is critical and - * HashFunctions where ease-of-use wins over. - */ +/** Type of hash function to use for digesting files. */ // The underlying HashFunctions are immutable and thread safe. public class DigestHashFunction { // This map must be declared first to make sure that calls to register() have it ready. @@ -67,12 +59,11 @@ public int getDigestMaximumLength() { public static final DigestHashFunction SHA1 = register(Hashing.sha1(), "SHA-1", "SHA1"); public static final DigestHashFunction SHA256 = register(Hashing.sha256(), "SHA-256", "SHA256"); + public static final DigestHashFunction BLAKE3 = register(new Blake3HashFunction(), "BLAKE3"); private final HashFunction hashFunction; private final DigestLength digestLength; private final String name; - private final MessageDigest messageDigestPrototype; - private final boolean messageDigestPrototypeSupportsClone; private final ImmutableList names; private DigestHashFunction( @@ -82,8 +73,6 @@ private DigestHashFunction( checkArgument(!names.isEmpty()); this.name = names.get(0); this.names = names; - this.messageDigestPrototype = getMessageDigestInstance(); - this.messageDigestPrototypeSupportsClone = supportsClone(messageDigestPrototype); } public static DigestHashFunction register( @@ -95,8 +84,7 @@ public static DigestHashFunction register( * Creates a new DigestHashFunction that is registered to be recognized by its name in {@link * DigestFunctionConverter}. * - * @param hashName the canonical name for this hash function - and the name that can be used to - * uncover the MessageDigest. + * @param hashName the canonical name for this hash function. * @param altNames alternative names that will be mapped to this function by the converter but * will not serve as the canonical name for the DigestHashFunction. * @param hash The {@link HashFunction} to register. @@ -104,15 +92,6 @@ public static DigestHashFunction register( */ public static DigestHashFunction register( HashFunction hash, DigestLength digestLength, String hashName, String... altNames) { - try { - MessageDigest.getInstance(hashName); - } catch (NoSuchAlgorithmException e) { - throw new IllegalArgumentException( - "The hash function name provided does not correspond to a valid MessageDigest: " - + hashName, - e); - } - ImmutableList names = ImmutableList.builder().add(hashName).add(altNames).build(); DigestHashFunction hashFunction = new DigestHashFunction(hash, digestLength, names); @@ -149,19 +128,6 @@ public HashFunction getHashFunction() { return hashFunction; } - public MessageDigest cloneOrCreateMessageDigest() { - if (messageDigestPrototypeSupportsClone) { - try { - return (MessageDigest) messageDigestPrototype.clone(); - } catch (CloneNotSupportedException e) { - // We checked at initialization that this could be cloned, so this should never happen. - throw new IllegalStateException("Could not clone message digest", e); - } - } else { - return getMessageDigestInstance(); - } - } - public DigestLength getDigestLength() { return digestLength; } @@ -175,25 +141,6 @@ public String toString() { return name; } - private MessageDigest getMessageDigestInstance() { - try { - return MessageDigest.getInstance(name); - } catch (NoSuchAlgorithmException e) { - // We check when we register() this digest function that the message digest exists. This - // should never happen. - throw new IllegalStateException("message digest " + name + " not available", e); - } - } - - private static boolean supportsClone(MessageDigest toCheck) { - try { - var unused = toCheck.clone(); - return true; - } catch (CloneNotSupportedException e) { - return false; - } - } - public static ImmutableSet getPossibleHashFunctions() { return ImmutableSet.copyOf(hashFunctionRegistry.values()); } diff --git a/src/main/native/BUILD b/src/main/native/BUILD index 2601d3298cfb30..a8e91c130c4c9a 100644 --- a/src/main/native/BUILD +++ b/src/main/native/BUILD @@ -54,9 +54,23 @@ cc_library( includes = ["."], # For jni headers. ) +cc_library( + name = "blake3_jni", + srcs = [ + "blake3_jni.cc", + ":jni.h", + ":jni_md.h", + ], + deps = [ + "@blake3//:blake3", + ], + includes = ["."], # For jni headers. +) + cc_binary( name = "libunix_jni.so", srcs = [ + "blake3_jni.cc", "macros.h", "process.cc", "unix_jni.cc", @@ -81,6 +95,7 @@ cc_binary( visibility = ["//src/main/java/com/google/devtools/build/lib/jni:__pkg__"], deps = [ ":latin1_jni_path", + ":blake3_jni", "//src/main/cpp/util:logging", "//src/main/cpp/util:md5", "//src/main/cpp/util:port", diff --git a/src/main/native/blake3_jni.cc b/src/main/native/blake3_jni.cc new file mode 100644 index 00000000000000..e2c32ccc8af79b --- /dev/null +++ b/src/main/native/blake3_jni.cc @@ -0,0 +1,80 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +#include "external/blake3/c/blake3.h" + +namespace blaze_jni { + +blake3_hasher *hasher_ptr(jlong self) { return (blake3_hasher *)self; } + +jbyte *get_byte_array(JNIEnv *env, jbyteArray java_array) { + return env->GetByteArrayElements(java_array, nullptr); +} + +void release_byte_array(JNIEnv *env, jbyteArray array, jbyte *addr) { + return env->ReleaseByteArrayElements(array, addr, 0); +} + +extern "C" JNIEXPORT jlong JNICALL +Java_com_google_devtools_build_lib_vfs_Blake3JNI_allocate_1and_1initialize_1hasher( + JNIEnv *env, jobject obj) { + blake3_hasher *hasher = new blake3_hasher; + blake3_hasher_init(hasher); + return (jlong)hasher; +} + +extern "C" JNIEXPORT void JNICALL +Java_com_google_devtools_build_lib_vfs_Blake3JNI_blake3_1hasher_1update( + JNIEnv *env, jobject obj, jlong self, jbyteArray input, jint input_len) { + + jbyte *input_addr = get_byte_array(env, input); + blake3_hasher_update(hasher_ptr(self), input_addr, input_len); + release_byte_array(env, input, input_addr); +} + +extern "C" JNIEXPORT void JNICALL +Java_com_google_devtools_build_lib_vfs_Blake3JNI_oneshot( + JNIEnv *env, jobject obj, jbyteArray input, jint input_len, jbyteArray out, + jint out_len) { + blake3_hasher *hasher = new blake3_hasher; + blake3_hasher_init(hasher); + + jbyte *input_addr = get_byte_array(env, input); + blake3_hasher_update(hasher, input_addr, input_len); + release_byte_array(env, input, input_addr); + + jbyte *out_addr = get_byte_array(env, out); + blake3_hasher_finalize(hasher, (uint8_t *)out_addr, out_len); + release_byte_array(env, out, out_addr); + + delete hasher; +} + +extern "C" JNIEXPORT void JNICALL +Java_com_google_devtools_build_lib_vfs_Blake3JNI_blake3_1hasher_1finalize_1and_1close( + JNIEnv *env, jobject obj, jlong self, jbyteArray out, jint out_len) { + blake3_hasher *hasher = hasher_ptr(self); + + jbyte *out_addr = get_byte_array(env, out); + blake3_hasher_finalize(hasher, (uint8_t *)out_addr, out_len); + release_byte_array(env, out, out_addr); + + delete hasher; +} + +} // namespace blaze_jni diff --git a/src/test/java/com/google/devtools/build/lib/vfs/Blake3HasherTest.java b/src/test/java/com/google/devtools/build/lib/vfs/Blake3HasherTest.java new file mode 100644 index 00000000000000..9c101ddfa25545 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/Blake3HasherTest.java @@ -0,0 +1,48 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.vfs; + +import static org.junit.Assert.assertEquals; + +import java.nio.charset.StandardCharsets; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for {@link Blake3Hasher}. */ +@RunWith(JUnit4.class) +public class Blake3HasherTest { + @Test + public void emptyHash() { + Blake3Hasher h = new Blake3Hasher(); + + byte[] data = new byte[0]; + h.putBytes(data); + + assertEquals( + "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", h.hash().toString()); + } + + @Test + public void helloWorld() { + Blake3Hasher h = new Blake3Hasher(); + + byte[] data = "hello world".getBytes(StandardCharsets.US_ASCII); + h.putBytes(data); + + assertEquals( + "d74981efa70a0c880b8d8c1985d075dbcbf679b99a5f9914e5aaf96b831a9e24", h.hash().toString()); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/vfs/DigestHashFunctionsTest.java b/src/test/java/com/google/devtools/build/lib/vfs/DigestHashFunctionsTest.java deleted file mode 100644 index 6b3b19923d2561..00000000000000 --- a/src/test/java/com/google/devtools/build/lib/vfs/DigestHashFunctionsTest.java +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2018 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package com.google.devtools.build.lib.vfs; - -import static com.google.common.truth.Truth.assertThat; - -import com.google.common.collect.ImmutableList; -import java.nio.charset.StandardCharsets; -import java.util.Collection; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameter; -import org.junit.runners.Parameterized.Parameters; - -/** - * Tests different {@link DigestHashFunction} for consistency between the MessageDigests and the - * HashFunctions that it exposes. - */ -@RunWith(Parameterized.class) -public class DigestHashFunctionsTest { - @Parameters(name = "{index}: digestHashFunction={0}") - public static Collection hashFunctions() { - // TODO(b/112537387): Remove the array-ification and return Collection. This - // is possible in Junit4.12, but 4.11 requires the array. Bazel 0.18 will have Junit4.12, so - // this can change then. - return DigestHashFunction.getPossibleHashFunctions() - .stream() - .map(dhf -> new DigestHashFunction[] {dhf}) - .collect(ImmutableList.toImmutableList()); - } - - @Parameter public DigestHashFunction digestHashFunction; - - private void assertHashFunctionAndMessageDigestEquivalentForInput(byte[] input) { - byte[] hashFunctionOutput = digestHashFunction.getHashFunction().hashBytes(input).asBytes(); - byte[] messageDigestOutput = digestHashFunction.cloneOrCreateMessageDigest().digest(input); - assertThat(hashFunctionOutput).isEqualTo(messageDigestOutput); - } - - @Test - public void emptyDigestIsConsistent() { - assertHashFunctionAndMessageDigestEquivalentForInput(new byte[] {}); - } - - @Test - public void shortDigestIsConsistent() { - assertHashFunctionAndMessageDigestEquivalentForInput("Bazel".getBytes(StandardCharsets.UTF_8)); - } -} diff --git a/third_party/blake3/blake3.BUILD b/third_party/blake3/blake3.BUILD index 6e96f051f1412e..aeec30d91b982a 100644 --- a/third_party/blake3/blake3.BUILD +++ b/third_party/blake3/blake3.BUILD @@ -56,10 +56,11 @@ cc_library( "-DBLAKE3_USE_NEON=1", ], "//conditions:default": [ - "-DBLAKE3_NO_SSE2", - "-DBLAKE3_NO_SSE41", "-DBLAKE3_NO_AVX2", "-DBLAKE3_NO_AVX512", + "-DBLAKE3_NO_NEON", + "-DBLAKE3_NO_SSE2", + "-DBLAKE3_NO_SSE41", ], }), includes = ["."],