diff --git a/BUCK b/BUCK index b3cb58ff6..061f9f02f 100644 --- a/BUCK +++ b/BUCK @@ -147,12 +147,27 @@ zs_library( ], ) +zs_library( + name = "dict", + srcs = glob([ + "src/openzl/dict/**/*.c", + ]), + headers = private_headers(glob([ + "src/openzl/dict/**/*.h", + ])), + header_namespace = "", + exported_deps = [ + ":common", + ], +) + zs_library( name = "zstronglib", exported_deps = [ ":common", ":compress", ":decompress", + ":dict", ], exported_external_deps = [ "zstd", diff --git a/include/openzl/detail/zl_errors_detail.h b/include/openzl/detail/zl_errors_detail.h index 897a29dcb..046342701 100644 --- a/include/openzl/detail/zl_errors_detail.h +++ b/include/openzl/detail/zl_errors_detail.h @@ -146,6 +146,10 @@ extern "C" { #define ZL_ErrorCode_srcSize_tooLarge__desc_str "Source size too large" #define ZL_ErrorCode_integerOverflow__desc_str "Integer overflow" #define ZL_ErrorCode_invalidName__desc_str "Invalid name of graph component" +#define ZL_ErrorCode_dict_corruption__desc_str \ + "Dictionary corruption: raw blob does not match expected wire format" +#define ZL_ErrorCode_dict_materialization__desc_str \ + "Dictionary materialization failed" /********************** * ZL_StaticErrorInfo * diff --git a/include/openzl/zl_errors_types.h b/include/openzl/zl_errors_types.h index b0dc77376..6f1247b44 100644 --- a/include/openzl/zl_errors_types.h +++ b/include/openzl/zl_errors_types.h @@ -89,6 +89,9 @@ typedef enum { ZL_ErrorCode_formatVersion_unsupported = 60, ZL_ErrorCode_formatVersion_notSet = 61, ZL_ErrorCode_node_versionMismatch = 62, + /* dictionary errors */ + ZL_ErrorCode_dict_corruption = 65, + ZL_ErrorCode_dict_materialization = 66, /* internal errors */ ZL_ErrorCode_allocation = 70, ZL_ErrorCode_internalBuffer_tooSmall = 71, diff --git a/src/openzl/common/errors.c b/src/openzl/common/errors.c index 57d5c24b0..4cb10e6ef 100644 --- a/src/openzl/common/errors.c +++ b/src/openzl/common/errors.c @@ -118,6 +118,10 @@ const char* ZL_ErrorCode_toString(ZL_ErrorCode code) return ZL_ErrorCode_srcSize_tooLarge__desc_str; case ZL_ErrorCode_integerOverflow: return ZL_ErrorCode_integerOverflow__desc_str; + case ZL_ErrorCode_dict_corruption: + return ZL_ErrorCode_dict_corruption__desc_str; + case ZL_ErrorCode_dict_materialization: + return ZL_ErrorCode_dict_materialization__desc_str; case ZL_ErrorCode_maxCode: default: ZL_ASSERT_FAIL("Invalid error code!: %d", (int)code); diff --git a/src/openzl/common/limits.h b/src/openzl/common/limits.h index 934d721f2..83ad5ad56 100644 --- a/src/openzl/common/limits.h +++ b/src/openzl/common/limits.h @@ -72,6 +72,10 @@ size_t ZL_transformOutStreamsLimit(unsigned formatVersion); /// Size limit for the variable sized comment field #define ZL_MAX_HEADER_COMMENT_SIZE_LIMIT 10000 +/// Registering more than this number of dictionaries on a single +/// compressor/dctx will fail. +#define ZL_DICTIONARY_LIMIT 1000 + //////////////////////////////////////// // Compressor Serialization Limits //////////////////////////////////////// diff --git a/src/openzl/dict/dict.c b/src/openzl/dict/dict.c new file mode 100644 index 000000000..46205d295 --- /dev/null +++ b/src/openzl/dict/dict.c @@ -0,0 +1,41 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#include "openzl/dict/dict.h" + +#include + +#include "openzl/common/errors_internal.h" + +const uint32_t ZL_DICT_MAGIC = 0x5A4C4449; // "ZLDI" + +ZL_RESULT_OF(ZL_ParsedDict) +ZL_Dict_parse( + ZL_OperationContext* opctx, + const void* dictBlob, + const size_t blobSize) +{ + ZL_ASSERT_NN(opctx); + ZL_RESULT_DECLARE_SCOPE(ZL_ParsedDict, opctx); + ZL_ERR_IF_NULL(dictBlob, dict_materialization, "dictBlob cannot be NULL"); + ZL_ERR_IF_LT( + blobSize, + 3 * sizeof(uint32_t), + dict_corruption, + "dict blob must contain metadata fields"); + ZL_ERR_IF_NE( + memcmp(dictBlob, &ZL_DICT_MAGIC, sizeof(ZL_DICT_MAGIC)), + 0, + dict_corruption, + "invalid dict magic"); + ZL_ParsedDict ret; + ret.codecId = ((const uint32_t*)dictBlob)[1]; + ret.dictSize = ((const uint32_t*)dictBlob)[2]; + ZL_ERR_IF_NE( + blobSize, + ret.dictSize + 3 * sizeof(uint32_t), + dict_corruption, + "Dict blob size mismatch"); + ret.rawDictContent = (const uint8_t*)dictBlob + 3 * sizeof(uint32_t); + ret.hash = ZL_SHA256_compute(ret.rawDictContent, ret.dictSize); + return ZL_WRAP_VALUE(ret); +} diff --git a/src/openzl/dict/dict.h b/src/openzl/dict/dict.h new file mode 100644 index 000000000..9e39499ba --- /dev/null +++ b/src/openzl/dict/dict.h @@ -0,0 +1,60 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#ifndef OPENZL_DICT_DICT_H +#define OPENZL_DICT_DICT_H + +#include "openzl/common/allocation.h" +#include "openzl/dict/sha256.h" +#include "openzl/zl_errors.h" +#include "openzl/zl_opaque_types.h" // ZL_NodeID + +#if defined(__cplusplus) +extern "C" { +#endif + +// ============================================================================ +// ZL_Dict - Single Dictionary Structure +// ============================================================================ + +// Intermediate structure useful for routing the dict to the correct node's +// materializer. Does NOT own any pointers. Ensure the raw dict blob outlives +// this struct. +typedef struct { + ZL_IDType codecId; + ZL_SHA256 hash; + uint32_t dictSize; + const void* rawDictContent; // NON-owning pointer +} ZL_ParsedDict; +ZL_RESULT_DECLARE_TYPE(ZL_ParsedDict); + +/** + * @brief Single dictionary structure + * + * NOTE: ZL_Dict is the MATERIALIZED dictionary object, not the serialized + * wire representation. It is created from raw dictionary bytes via + * ZL_Dict_create() or via context-specific materialization functions. + */ +typedef struct { + ZL_IDType codecId; // Codec responsible for materializing/dematerializing + ZL_SHA256 hash; // Precomputed SHA-256 hash of dictContent + void (*dematerializeFn)(void* dictObj); + void* dictObj; // Materialized data (caller-allocated or + // arena-allocated) +} ZL_Dict; + +/** + * Does basic validation and generates an intermediate representation of the + * dict blob. Used by the ZL_Compressor and ZL_DCtx, which are expected to then + * call the proper node materializer to generate the full ZL_Dict structure. + */ +ZL_RESULT_OF(ZL_ParsedDict) +ZL_Dict_parse( + ZL_OperationContext* opctx, + const void* dictBlob, + const size_t blobSize); + +#if defined(__cplusplus) +} // extern "C" +#endif + +#endif // OPENZL_DICT_DICT_H diff --git a/src/openzl/dict/dictbundle.c b/src/openzl/dict/dictbundle.c new file mode 100644 index 000000000..03aefb62e --- /dev/null +++ b/src/openzl/dict/dictbundle.c @@ -0,0 +1,82 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#include "openzl/dict/dictbundle.h" + +#include +#include + +#include "openzl/common/allocation.h" + +const uint32_t ZL_DICTBUNDLE_COMP_MAGIC = + 0x5A4C4243; // "ZLBC" (compression-only) +const uint32_t ZL_DICTBUNDLE_BIDI_MAGIC = 0x5A4C4249; // "ZLBI" (bidirectional) + +// ============================================================================ +// ZL_DictBundle Implementation +// ============================================================================ + +ZL_RESULT_OF(ZL_DictBundlePtr) +DictBundle_create( + ZL_OperationContext* opctx, + Arena* arena, + const void* dictBlob, + const size_t blobSize) +{ + ZL_RESULT_DECLARE_SCOPE(ZL_DictBundlePtr, opctx); + ZL_ASSERT_NN(arena); + ZL_ASSERT_NN(dictBlob); + + ZL_ERR_IF_LT( + blobSize, + sizeof(uint32_t) * 2 + sizeof(ZL_SHA256), + dict_materialization, + "DictBundle blob too small for header"); + + const char* ptr = (const char*)dictBlob; + + uint32_t magic; + memcpy(&magic, ptr, sizeof(uint32_t)); + ptr += sizeof(uint32_t); + + ZL_DictBundle* bundle = ALLOC_Arena_malloc(arena, sizeof(ZL_DictBundle)); + ZL_ERR_IF_NULL(bundle, allocation, "Failed to allocate ZL_DictBundle"); + if (magic == ZL_DICTBUNDLE_COMP_MAGIC) { + bundle->compressionOnly = true; + } else if (magic == ZL_DICTBUNDLE_BIDI_MAGIC) { + bundle->compressionOnly = false; + } else { + ZL_ERR(dict_corruption, "Invalid dict bundle magic"); + } + + memcpy(&bundle->bundleId, ptr, sizeof(ZL_SHA256)); + ptr += sizeof(ZL_SHA256); + + memcpy(&bundle->numDicts, ptr, sizeof(uint32_t)); + ptr += sizeof(uint32_t); + + ZL_ERR_IF_NE( + blobSize, + sizeof(uint32_t) * 2 + sizeof(ZL_SHA256) + + bundle->numDicts * sizeof(ZL_SHA256), + dict_corruption, + "DictBundle blob size mismatch"); + + bundle->dicts = + ALLOC_Arena_malloc(arena, sizeof(ZL_SHA256) * bundle->numDicts); + ZL_ERR_IF_NULL( + bundle->dicts, + allocation, + "Failed to allocate ZL_DictBundle dicts"); + memcpy(bundle->dicts, ptr, sizeof(ZL_SHA256) * bundle->numDicts); + + return ZL_WRAP_VALUE(bundle); +} + +void DictBundle_free(Arena* arena, ZL_DictBundle* bundle) +{ + if (bundle == NULL) { + return; + } + ALLOC_Arena_free(arena, bundle->dicts); + ALLOC_Arena_free(arena, bundle); +} diff --git a/src/openzl/dict/dictbundle.h b/src/openzl/dict/dictbundle.h new file mode 100644 index 000000000..5da6b9c96 --- /dev/null +++ b/src/openzl/dict/dictbundle.h @@ -0,0 +1,45 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#ifndef OPENZL_DICT_DICTBUNDLE_H +#define OPENZL_DICT_DICTBUNDLE_H + +#include // size_t + +#include "openzl/common/allocation.h" +#include "openzl/common/errors_internal.h" +#include "openzl/dict/sha256.h" + +#if defined(__cplusplus) +extern "C" { +#endif + +typedef struct { + ZL_SHA256 bundleId; + bool compressionOnly; + size_t numDicts; + ZL_SHA256* dicts; +} ZL_DictBundle; + +typedef ZL_DictBundle* ZL_DictBundlePtr; +ZL_RESULT_DECLARE_TYPE(ZL_DictBundlePtr); + +size_t ZL_DictBundle_numDicts(const ZL_DictBundle* bundle); +ZL_SHA256* ZL_DictBundle_dictHashes(const ZL_DictBundle* bundle); + +/** + * Parse and allocate a bundle from the raw blob. Does safety checks. + */ +ZL_RESULT_OF(ZL_DictBundlePtr) +DictBundle_create( + ZL_OperationContext* opctx, + Arena* arena, + const void* dictBlob, + const size_t blobSize); + +void DictBundle_free(Arena* arena, ZL_DictBundle* bundle); + +#if defined(__cplusplus) +} // extern "C" +#endif + +#endif // OPENZL_DICT_DICTBUNDLE_H diff --git a/src/openzl/dict/dictstore.c b/src/openzl/dict/dictstore.c new file mode 100644 index 000000000..6723aa945 --- /dev/null +++ b/src/openzl/dict/dictstore.c @@ -0,0 +1,250 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#include "openzl/dict/dictstore.h" + +#include "openzl/common/allocation.h" +#include "openzl/common/errors_internal.h" +#include "openzl/common/limits.h" +#include "openzl/dict/dict.h" +#include "openzl/dict/dictbundle.h" +#include "openzl/dict/sha256.h" +#include "openzl/zl_errors.h" +#include "openzl/zl_opaque_types.h" // ZL_IDType + +ZL_DictStore* ZL_DictStore_create(void) +{ + ZL_DictStore* store = ZL_malloc(sizeof(ZL_DictStore)); + if (store == NULL) { + return NULL; + } + + store->arena = ALLOC_HeapArena_create(); + + store->bundles = + ZL_BundleMap_createInArena(store->arena, ZL_DICTIONARY_LIMIT); + store->dicts = ZL_DictMap_createInArena(store->arena, ZL_DICTIONARY_LIMIT); + + return store; +} + +void ZL_DictStore_reset(ZL_DictStore* store) +{ + if (store == NULL) { + return; + } + + // Dematerialize all dicts + ZL_DictMap_Iter dictIter = ZL_DictMap_iter(&store->dicts); + ZL_DictMap_Entry const* dictEntry; + while ((dictEntry = ZL_DictMap_Iter_next(&dictIter)) != NULL) { + ZL_Dict* dict = dictEntry->val; + dict->dematerializeFn(dict->dictObj); + } + + ALLOC_Arena_freeAll(store->arena); + + // Recreate the maps with fresh allocations + store->bundles = + ZL_BundleMap_createInArena(store->arena, ZL_DICTIONARY_LIMIT); + store->dicts = ZL_DictMap_createInArena(store->arena, ZL_DICTIONARY_LIMIT); +} + +void ZL_DictStore_free(ZL_DictStore* store) +{ + if (store == NULL) { + return; + } + + // Dematerialize all dicts + ZL_DictMap_Iter dictIter = ZL_DictMap_iter(&store->dicts); + ZL_DictMap_Entry const* dictEntry; + while ((dictEntry = ZL_DictMap_Iter_next(&dictIter)) != NULL) { + ZL_Dict* dict = dictEntry->val; + dict->dematerializeFn(dict->dictObj); + } + ALLOC_Arena_freeAll(store->arena); + ZL_free(store); +} + +// ==================== ZL_Dict manip =================== +/** + * Materialize and store a dict within the dict store. Requires either a + * compressor or dctx to do the materialization + */ +ZL_RESULT_OF(ZL_SHA256) +ZL_DictStore_matererializeDict( + ZL_DictStore* store, + ZL_Compressor* compressor, + const void* rawDict, + size_t rawDictSize) +{ + ZL_RESULT_DECLARE_SCOPE(ZL_SHA256, NULL); + + ZL_ASSERT_NN(store, "store cannot be NULL"); + ZL_ASSERT_NN(compressor, "compressor cannot be NULL"); + ZL_ASSERT_NN(rawDict, "rawDict cannot be NULL"); + ZL_ASSERT_GT(rawDictSize, 0, "rawDictSize cannot be 0"); + + ZL_RESULT_OF(ZL_ParsedDict) + parsedRes = ZL_Dict_parse(NULL, rawDict, rawDictSize); + ZL_ERR_IF_ERR(parsedRes); + ZL_ParsedDict parsed = ZL_RES_value(parsedRes); + + // Check if dict already exists + if (ZL_DictMap_find(&store->dicts, &parsed.hash)) { + return ZL_WRAP_VALUE(parsed.hash); + } + + // Allocate and initialize ZL_Dict structure + ZL_Dict* dict = ALLOC_Arena_malloc(store->arena, sizeof(ZL_Dict)); + ZL_ERR_IF_NULL(dict, allocation, "Failed to allocate ZL_Dict"); + + dict->codecId = parsed.codecId; + dict->hash = parsed.hash; + // TODO(csv): next diff, call ZL_Compressor_getMaterializer(ZL_SHA256) + // 3. Double-check that the compressor has a record of mapping that codec to + // the SHA of the dict + // 4. Materialize using the codec's materialization function and add it to + // the dict store + dict->dematerializeFn = NULL; + dict->dictObj = NULL; + + // Add to store + ZL_DictMap_Entry entry = { .key = parsed.hash, .val = dict }; + ZL_DictMap_Insert insertResult = ZL_DictMap_insert(&store->dicts, &entry); + if (insertResult.badAlloc) { + ALLOC_Arena_free(store->arena, dict); + ZL_ERR(allocation, "Failed to insert dict into store map"); + } + + return ZL_WRAP_VALUE(parsed.hash); +} + +ZL_RESULT_OF(ZL_SHA256) +ZL_DictStore_materializeDict( + ZL_DictStore* store, + ZL_DCtx* dctx, + const void* rawDict, + size_t rawDictSize) +{ + ZL_RESULT_DECLARE_SCOPE(ZL_SHA256, NULL); + + ZL_ASSERT_NN(store, "store cannot be NULL"); + ZL_ASSERT_NN(dctx, "compressor cannot be NULL"); + ZL_ASSERT_NN(rawDict, "rawDict cannot be NULL"); + + ZL_RESULT_OF(ZL_ParsedDict) + parsedRes = ZL_Dict_parse(NULL, rawDict, rawDictSize); + ZL_ERR_IF_ERR(parsedRes); + ZL_ParsedDict parsed = ZL_RES_value(parsedRes); + + // Check if dict already exists + if (ZL_DictStore_hasDict(store, parsed.hash)) { + return ZL_WRAP_VALUE(parsed.hash); + } + + // Allocate and initialize ZL_Dict structure + ZL_Dict* dict = ALLOC_Arena_malloc(store->arena, sizeof(ZL_Dict)); + ZL_ERR_IF_NULL(dict, allocation, "Failed to allocate ZL_Dict"); + + dict->codecId = parsed.codecId; + dict->hash = parsed.hash; + // TODO(csv): next diff, call ZL_Compressor_getMaterializer(ZL_SHA256) + // 3. Double-check that the compressor has a record of mapping that codec to + // the SHA of the dict + // 4. Materialize using the codec's materialization function and add it to + // the dict store + dict->dematerializeFn = NULL; + dict->dictObj = NULL; + + // Add to store + ZL_DictMap_Entry entry = { .key = parsed.hash, .val = dict }; + ZL_DictMap_Insert insertResult = ZL_DictMap_insert(&store->dicts, &entry); + if (insertResult.badAlloc) { + dict->dematerializeFn(dict->dictObj); + ALLOC_Arena_free(store->arena, dict); + ZL_ERR(allocation, "Failed to insert dict into store map"); + } + + return ZL_WRAP_VALUE(parsed.hash); +} + +bool ZL_DictStore_hasDict(ZL_DictStore* store, ZL_SHA256 dictID) +{ + if (store == NULL) { + return false; + } + return ZL_DictMap_find(&store->dicts, &dictID) != NULL; +} + +// ================= ZL_DictBundle manip ================ + +ZL_RESULT_OF(ZL_SHA256) +ZL_DictStore_materializeBundle( + ZL_DictStore* store, + void* rawBundle, + size_t rawBundleSize) +{ + ZL_RESULT_DECLARE_SCOPE(ZL_SHA256, NULL); + + ZL_ASSERT_NN(store, "store cannot be NULL"); + ZL_ASSERT_NN(rawBundle, "rawBundle cannot be NULL"); + + ZL_RESULT_OF(ZL_DictBundlePtr) + bundleRes = DictBundle_create(NULL, store->arena, rawBundle, rawBundleSize); + ZL_ERR_IF_ERR(bundleRes); + ZL_DictBundle* bundle = ZL_RES_value(bundleRes); + + // Check if bundle already exists + if (ZL_DictStore_hasBundle(store, bundle->bundleId)) { + DictBundle_free(store->arena, bundle); + return ZL_WRAP_VALUE(bundle->bundleId); + } + + // Insert the bundle into the store's map + ZL_BundleMap_Entry entry = { .key = bundle->bundleId, .val = bundle }; + ZL_BundleMap_Insert insertResult = + ZL_BundleMap_insert(&store->bundles, &entry); + if (insertResult.badAlloc) { + DictBundle_free(store->arena, bundle); + ZL_ERR(allocation, "Failed to insert bundle into store map"); + } + return ZL_WRAP_VALUE(bundle->bundleId); +} + +bool ZL_DictStore_hasBundle(ZL_DictStore* store, ZL_SHA256 bundleID) +{ + if (store == NULL) { + return false; + } + return ZL_BundleMap_find(&store->bundles, &bundleID) != NULL; +} + +ZL_DictBundle* ZL_DictStore_getBundle(ZL_DictStore* store, ZL_SHA256 bundleID) +{ + if (store == NULL) { + return NULL; + } + const ZL_BundleMap_Entry* entry = + ZL_BundleMap_find(&store->bundles, &bundleID); + return entry != NULL ? entry->val : NULL; +} + +bool ZL_DictStore_bundleIsComplete(ZL_DictStore* store, ZL_SHA256 bundleID) +{ + if (store == NULL) { + return false; + } + + ZL_DictBundle* bundle = ZL_DictStore_getBundle(store, bundleID); + if (bundle == NULL) { + return false; + } + + for (uint32_t i = 0; i < bundle->numDicts; i++) { + if (!ZL_DictStore_hasDict(store, bundle->dicts[i])) { + return false; + } + } + return true; +} diff --git a/src/openzl/dict/dictstore.h b/src/openzl/dict/dictstore.h new file mode 100644 index 000000000..7383a859f --- /dev/null +++ b/src/openzl/dict/dictstore.h @@ -0,0 +1,108 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#ifndef OPENZL_DICT_DICTSTORE_H +#define OPENZL_DICT_DICTSTORE_H + +#include "openzl/common/allocation.h" +#include "openzl/common/map.h" +#include "openzl/dict/dict.h" +#include "openzl/dict/dictbundle.h" +#include "openzl/dict/sha256.h" +#include "openzl/zl_errors.h" +#include "openzl/zl_opaque_types.h" // ZL_NodeID + +#if defined(__cplusplus) +extern "C" { +#endif + +ZL_DECLARE_PREDEF_MAP_TYPE(ZL_BundleMap, ZL_SHA256, ZL_DictBundle*); +ZL_DECLARE_PREDEF_MAP_TYPE(ZL_DictMap, ZL_SHA256, ZL_Dict*); + +typedef struct { + Arena* arena; + ZL_BundleMap bundles; + ZL_DictMap dicts; +} ZL_DictStore; + +/** + * Create a new ZL_DictStore with an arena. + */ +ZL_DictStore* ZL_DictStore_create(void); + +/** + * Clears the arena and frees *all* dicts using the dematerialization functions. + */ +void ZL_DictStore_reset(ZL_DictStore* store); +// also kills the struct +void ZL_DictStore_free(ZL_DictStore* store); + +// ==================== ZL_Dict manip =================== + +/** + * Add an already-materialized dict to the store. + * Helper function for unit testing. + */ +ZL_RESULT_OF(ZL_SHA256) +DictStore_addDict(ZL_DictStore* store, ZL_Dict* dict); + +/** + * Lookup a dict by its hash. + * @returns Pointer to the dict if found, NULL otherwise. + */ +ZL_Dict* DictStore_getDict(ZL_DictStore* store, ZL_SHA256 dictID); + +bool ZL_DictStore_hasDict(ZL_DictStore* store, ZL_SHA256 dictID); + +/** + * Materialize and store a dict within the dict store. Requires either a + * compressor or dctx to do the materialization + */ +ZL_RESULT_OF(ZL_SHA256) +ZL_DictStore_matererializeDict( + ZL_DictStore* store, + ZL_Compressor* compressor, + const void* rawDict, + size_t rawDictSize); + +ZL_RESULT_OF(ZL_SHA256) +ZL_DictStore_materializeDict( + ZL_DictStore* store, + ZL_DCtx* dctx, + const void* rawDict, + size_t rawDictSize); + +// ================= ZL_DictBundle manip ================ + +/** + * Add an already-materialized bundle to the store. + * Helper function for unit testing. + */ +ZL_RESULT_OF(ZL_SHA256) +DictStore_addBundle(ZL_DictStore* store, ZL_DictBundle* bundle); + +/** + * Materialize a bundle and store it in the dict store + */ +ZL_RESULT_OF(ZL_SHA256) +ZL_DictStore_materializeBundle( + ZL_DictStore* store, + void* rawBundle, + size_t rawBundleSize); + +// map lookup +bool ZL_DictStore_hasBundle(ZL_DictStore* store, ZL_SHA256 bundleID); + +// return NULL if not there +ZL_DictBundle* ZL_DictStore_getBundle(ZL_DictStore* store, ZL_SHA256 bundleID); + +/** + * Whether the store contains all the dicts required for the bundle with ID @p + * bundleID + */ +bool ZL_DictStore_bundleIsComplete(ZL_DictStore* store, ZL_SHA256 bundleID); + +#if defined(__cplusplus) +} // extern "C" +#endif + +#endif // OPENZL_DICT_DICTSTORE_H diff --git a/src/openzl/dict/sha256.c b/src/openzl/dict/sha256.c new file mode 100644 index 000000000..89c52beb5 --- /dev/null +++ b/src/openzl/dict/sha256.c @@ -0,0 +1,170 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#include "openzl/dict/sha256.h" + +#include //size_t +#include //uint32_t, uint64_t +#include //memcpy, memcp + +// forward decl +void SHA256(const unsigned char* data, size_t len, unsigned char* digest); + +size_t ZL_SHA256_hash(const ZL_SHA256* key) +{ + if (key == NULL) { + return 0; + } + // SHA-256 already has excellent distribution, so we can just use the + // first sizeof(size_t) bytes directly as the hash value + size_t result; + memcpy(&result, key->bytes, sizeof(result)); + return result; +} + +bool ZL_SHA256_eq(const ZL_SHA256* lhs, const ZL_SHA256* rhs) +{ + if (lhs == NULL || rhs == NULL) { + return false; + } + return memcmp(lhs->bytes, rhs->bytes, sizeof(lhs->bytes)) == 0; +} + +ZL_SHA256 ZL_SHA256_compute(const void* data, size_t size) +{ + ZL_SHA256 result; + SHA256(data, size, result.bytes); + return result; +} + +bool ZL_SHA256_isValid(const ZL_SHA256* hash) +{ + if (hash == NULL) { + return false; + } + // Check if any byte is non-zero + for (size_t i = 0; i < sizeof(hash->bytes); i++) { + if (hash->bytes[i] != 0) { + return true; + } + } + return false; +} + +ZL_SHA256 ZL_SHA256_zero(void) +{ + ZL_SHA256 result; + memset(result.bytes, 0, sizeof(result.bytes)); + return result; +} + +// ============================================================================ +// ZL_SHA256 Implementation +// ============================================================================ +// TODO(csv): fast sha256 implementation + +// macros +#define S(x, n) \ + (((((uint32_t)(x) & 0xFFFFFFFFUL) >> (uint32_t)((n) & 31)) \ + | ((uint32_t)(x) << (uint32_t)((32 - ((n) & 31)) & 31))) \ + & 0xFFFFFFFFUL) +#define R(x, n) (((x) & 0xFFFFFFFFUL) >> (n)) +#define Gamma0(x) (S(x, 7) ^ S(x, 18) ^ R(x, 3)) +#define Gamma1(x) (S(x, 17) ^ S(x, 19) ^ R(x, 10)) +#define RND(a, b, c, d, e, f, g, h, i) \ + t0 = h + (S(e, 6) ^ S(e, 11) ^ S(e, 25)) + (g ^ (e & (f ^ g))) + K[i] \ + + W[i]; \ + t1 = (S(a, 2) ^ S(a, 13) ^ S(a, 22)) + (((a | b) & c) | (a & b)); \ + d += t0; \ + h = t0 + t1; +#define STORE32H(x, y) \ + (y)[0] = (unsigned char)(((x) >> 24) & 255); \ + (y)[1] = (unsigned char)(((x) >> 16) & 255); \ + (y)[2] = (unsigned char)(((x) >> 8) & 255); \ + (y)[3] = (unsigned char)((x) & 255); +#define LOAD32H(x, y) \ + x = ((uint32_t)((y)[0] & 255) << 24) | ((uint32_t)((y)[1] & 255) << 16) \ + | ((uint32_t)((y)[2] & 255) << 8) | ((uint32_t)((y)[3] & 255)); +#define STORE64H(x, y) \ + (y)[0] = (unsigned char)(((x) >> 56) & 255); \ + (y)[1] = (unsigned char)(((x) >> 48) & 255); \ + (y)[2] = (unsigned char)(((x) >> 40) & 255); \ + (y)[3] = (unsigned char)(((x) >> 32) & 255); \ + (y)[4] = (unsigned char)(((x) >> 24) & 255); \ + (y)[5] = (unsigned char)(((x) >> 16) & 255); \ + (y)[6] = (unsigned char)(((x) >> 8) & 255); \ + (y)[7] = (unsigned char)((x) & 255); +#define SHA256_COMPRESS(buff) \ + for (int i = 0; i < 8; i++) \ + S[i] = sha256_state[i]; \ + for (int i = 0; i < 16; i++) \ + LOAD32H(W[i], buff + (4 * i)); \ + for (int i = 16; i < 64; i++) \ + W[i] = Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]; \ + for (int i = 0; i < 64; i++) { \ + RND(S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], i); \ + t = S[7]; \ + S[7] = S[6]; \ + S[6] = S[5]; \ + S[5] = S[4]; \ + S[4] = S[3]; \ + S[3] = S[2]; \ + S[2] = S[1]; \ + S[1] = S[0]; \ + S[0] = t; \ + } \ + for (int i = 0; i < 8; i++) \ + sha256_state[i] = sha256_state[i] + S[i]; + +void SHA256(const unsigned char* in, size_t len, unsigned char* out) +{ + const uint32_t K[64] = { + 0x428a2f98UL, 0x71374491UL, 0xb5c0fbcfUL, 0xe9b5dba5UL, 0x3956c25bUL, + 0x59f111f1UL, 0x923f82a4UL, 0xab1c5ed5UL, 0xd807aa98UL, 0x12835b01UL, + 0x243185beUL, 0x550c7dc3UL, 0x72be5d74UL, 0x80deb1feUL, 0x9bdc06a7UL, + 0xc19bf174UL, 0xe49b69c1UL, 0xefbe4786UL, 0x0fc19dc6UL, 0x240ca1ccUL, + 0x2de92c6fUL, 0x4a7484aaUL, 0x5cb0a9dcUL, 0x76f988daUL, 0x983e5152UL, + 0xa831c66dUL, 0xb00327c8UL, 0xbf597fc7UL, 0xc6e00bf3UL, 0xd5a79147UL, + 0x06ca6351UL, 0x14292967UL, 0x27b70a85UL, 0x2e1b2138UL, 0x4d2c6dfcUL, + 0x53380d13UL, 0x650a7354UL, 0x766a0abbUL, 0x81c2c92eUL, 0x92722c85UL, + 0xa2bfe8a1UL, 0xa81a664bUL, 0xc24b8b70UL, 0xc76c51a3UL, 0xd192e819UL, + 0xd6990624UL, 0xf40e3585UL, 0x106aa070UL, 0x19a4c116UL, 0x1e376c08UL, + 0x2748774cUL, 0x34b0bcb5UL, 0x391c0cb3UL, 0x4ed8aa4aUL, 0x5b9cca4fUL, + 0x682e6ff3UL, 0x748f82eeUL, 0x78a5636fUL, 0x84c87814UL, 0x8cc70208UL, + 0x90befffaUL, 0xa4506cebUL, 0xbef9a3f7UL, 0xc67178f2UL + }; + uint64_t sha256_length = 0; + uint32_t sha256_state[8] = { 0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL, + 0xA54FF53AUL, 0x510E527FUL, 0x9B05688CUL, + 0x1F83D9ABUL, 0x5BE0CD19UL }, + S[8], W[64], t0, t1, t; + unsigned char sha256_buf[64]; + // process input in 64 byte chunks + while (len >= 64) { + SHA256_COMPRESS(in); + sha256_length += 64 * 8; + in += 64; + len -= 64; + } + // copy remaining bytes into sha256_buf + memcpy(sha256_buf, in, len); + // finish up (len now number of bytes in sha256_buf) + sha256_length += len * 8; + sha256_buf[len++] = 0x80; + // pad then compress if length is above 56 bytes + if (len > 56) { + while (len < 64) + sha256_buf[len++] = 0; + SHA256_COMPRESS(sha256_buf); + len = 0; + } + // pad up to 56 bytes + while (len < 56) + sha256_buf[len++] = 0; + // store length and compress + STORE64H(sha256_length, sha256_buf + 56); + SHA256_COMPRESS(sha256_buf); + // copy output + for (int i = 0; i < 8; i++) { + STORE32H(sha256_state[i], out + 4 * i); + } +} diff --git a/src/openzl/dict/sha256.h b/src/openzl/dict/sha256.h new file mode 100644 index 000000000..ad7972a49 --- /dev/null +++ b/src/openzl/dict/sha256.h @@ -0,0 +1,65 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#ifndef OPENZL_DICT_SHA256_H +#define OPENZL_DICT_SHA256_H + +#include // bool +#include // size_t + +#include "openzl/zl_errors.h" + +#if defined(__cplusplus) +extern "C" { +#endif + +// ============================================================================ +// ZL_SHA256 - SHA-256 Hash Type +// ============================================================================ + +/** + * @brief SHA-256 hash type (32 bytes) + * Used for dictionary uniqueness identification. + */ +typedef struct { + unsigned char bytes[32]; +} ZL_SHA256; + +ZL_RESULT_DECLARE_TYPE(ZL_SHA256); + +/** + * "Hash" function for ZL_SHA256 (required for map usage). Returns the top + * sizeof(size_t) bytes + */ +size_t ZL_SHA256_hash(const ZL_SHA256* key); + +bool ZL_SHA256_eq(const ZL_SHA256* lhs, const ZL_SHA256* rhs); + +/** + * @brief Compute SHA-256 hash of data + * + * @param data Pointer to the data to hash + * @param size Size of the data in bytes + * @returns Computed SHA-256 hash + */ +ZL_SHA256 ZL_SHA256_compute(const void* data, size_t size); + +/** + * @brief Check if a SHA-256 hash is valid (non-zero) + * + * @param hash Pointer to the hash to check + * @returns true if the hash is non-zero, false if all zeros + */ +bool ZL_SHA256_isValid(const ZL_SHA256* hash); + +/** + * @brief Get a zero-initialized SHA-256 hash + * + * @returns A SHA-256 hash with all bytes set to zero + */ +ZL_SHA256 ZL_SHA256_zero(void); + +#if defined(__cplusplus) +} // extern "C" +#endif + +#endif // OPENZL_DICT_SHA256_H diff --git a/src/openzl/dict/wire_format.md b/src/openzl/dict/wire_format.md new file mode 100644 index 000000000..d72814a6b --- /dev/null +++ b/src/openzl/dict/wire_format.md @@ -0,0 +1,15 @@ +# Wire and Materialized Format for ZL_Dict and ZL_DictBundle + +## ZL_Dict +- 32-bit little-endian (magic = 0x5A4C4449 "ZLDI") +- 32-bit little-endian (nodeId = materializer node ID, or UINT32_MAX if unbound) +- 32-bit little-endian (dictSize = length of content in bytes) +- char array (content, dictSize bytes) + +This is DIFFERENT from the in-memory representation, which contains, among other things, the hydrated dict object. + +## ZL_DictBundle +- 32-bit little-endian (magic) +- SHA-256 (unique ID) +- 32-bit little-endian (number dicts) +- SHA-256 array (dict SHAs)