diff --git a/Makefile.in b/Makefile.in index ccb3c4f099..c02c2c4459 100644 --- a/Makefile.in +++ b/Makefile.in @@ -164,6 +164,9 @@ crc32.o: $(SRCDIR)crc32.c crc32_vx.o: $(SRCDIR)contrib/crc32vx/crc32_vx.c $(CC) $(CFLAGS) $(VGFMAFLAG) $(ZINC) -c -o $@ $(SRCDIR)contrib/crc32vx/crc32_vx.c +dfltcc.o: $(SRCDIR)contrib/dfltcc/dfltcc.c + $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)contrib/dfltcc/dfltcc.c + deflate.o: $(SRCDIR)deflate.c $(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)deflate.c @@ -219,6 +222,11 @@ crc32_vx.lo: $(SRCDIR)contrib/crc32vx/crc32_vx.c $(CC) $(SFLAGS) $(VGFMAFLAG) $(ZINC) -DPIC -c -o objs/crc32_vx.o $(SRCDIR)contrib/crc32vx/crc32_vx.c -@mv objs/crc32_vx.o $@ +dfltcc.lo: $(SRCDIR)contrib/dfltcc/dfltcc.c + -@mkdir objs 2>/dev/null || test -d objs + $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/dfltcc.o $(SRCDIR)contrib/dfltcc/dfltcc.c + -@mv objs/dfltcc.o $@ + deflate.lo: $(SRCDIR)deflate.c -@mkdir objs 2>/dev/null || test -d objs $(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/deflate.o $(SRCDIR)deflate.c @@ -292,6 +300,9 @@ placebo $(SHAREDLIBV): $(PIC_OBJS) libz.a $(SRCDIR)zlib.map ln -s $@ $(SHAREDLIBM) -@rmdir objs +crc32_test$(EXE): crc32_test.o $(STATICLIB) + $(CC) $(CFLAGS) -o $@ crc32_test.o $(TEST_LIBS) + example$(EXE): example.o $(STATICLIB) $(CC) $(CFLAGS) $(LDFLAGS) -o $@ example.o $(TEST_LIBS) @@ -304,6 +315,9 @@ examplesh$(EXE): example.o $(SHAREDLIBV) minigzipsh$(EXE): minigzip.o $(SHAREDLIBV) $(CC) $(CFLAGS) -o $@ minigzip.o $(LDFLAGS) -L. $(SHAREDLIBV) +crc32_test64$(EXE): crc32_test64.o $(STATICLIB) + $(CC) $(CFLAGS) -o $@ crc32_test64.o $(TEST_LIBS) + example64$(EXE): example64.o $(STATICLIB) $(CC) $(CFLAGS) $(LDFLAGS) -o $@ example64.o $(TEST_LIBS) @@ -405,22 +419,28 @@ adler32.o: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h zutil.o: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)gzguts.h gzclose.o gzlib.o gzread.o gzwrite.o: $(SRCDIR)zlib.h zconf.h $(SRCDIR)gzguts.h compress.o example.o minigzip.o uncompr.o: $(SRCDIR)zlib.h zconf.h +compress.o: $(SRCDIR)contrib/dfltcc/hooks.h crc32.o: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)crc32.h -deflate.o: $(SRCDIR)deflate.h $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h +deflate.o: $(SRCDIR)deflate.h $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)contrib/dfltcc/hooks.h infback.o inflate.o: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)inftrees.h $(SRCDIR)inflate.h $(SRCDIR)inffast.h $(SRCDIR)inffixed.h +inflate.o: $(SRCDIR)contrib/dfltcc/hooks.h inffast.o: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)inftrees.h $(SRCDIR)inflate.h $(SRCDIR)inffast.h inftrees.o: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)inftrees.h trees.o: $(SRCDIR)deflate.h $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)trees.h crc32_vx.o: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)contrib/crc32vx/crc32_vx_hooks.h +dfltcc.o: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)contrib/dfltcc/dfltcc_hooks.h $(SRCDIR)contrib/dfltcc/dfltcc.h adler32.lo: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h zutil.lo: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)gzguts.h gzclose.lo gzlib.lo gzread.lo gzwrite.lo: $(SRCDIR)zlib.h zconf.h $(SRCDIR)gzguts.h compress.lo example.lo minigzip.lo uncompr.lo: $(SRCDIR)zlib.h zconf.h +compress.lo: $(SRCDIR)contrib/dfltcc/hooks.h crc32.lo: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)crc32.h -deflate.lo: $(SRCDIR)deflate.h $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h +deflate.lo: $(SRCDIR)deflate.h $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)contrib/dfltcc/hooks.h infback.lo inflate.lo: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)inftrees.h $(SRCDIR)inflate.h $(SRCDIR)inffast.h $(SRCDIR)inffixed.h +inflate.lo: $(SRCDIR)contrib/dfltcc/hooks.h inffast.lo: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)inftrees.h $(SRCDIR)inflate.h $(SRCDIR)inffast.h inftrees.lo: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)inftrees.h trees.lo: $(SRCDIR)deflate.h $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)trees.h -crc32_vx.lo: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)contrib/crc32vx/crc32_vx_hooks.h \ No newline at end of file +crc32_vx.lo: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)contrib/crc32vx/crc32_vx_hooks.h +dfltcc.lo: $(SRCDIR)zutil.h $(SRCDIR)zlib.h zconf.h $(SRCDIR)contrib/dfltcc/dfltcc_hooks.h $(SRCDIR)contrib/dfltcc/dfltcc.h \ No newline at end of file diff --git a/compress.c b/compress.c index bd74b9488e..0c01f6f732 100644 --- a/compress.c +++ b/compress.c @@ -7,6 +7,13 @@ #define ZLIB_INTERNAL #include "zlib.h" +#ifdef HAVE_S390X_DFLTCC +# include "contrib/dfltcc/dfltcc_hooks.h" +#else +# include "contrib/dfltcc/hooks.h" +#endif + +#define ZLIB_WRAPLEN 6 /* zlib format overhead */ /* =========================================================================== Compresses the source buffer into the destination buffer. The level @@ -94,6 +101,12 @@ z_size_t ZEXPORT compressBound_z(z_size_t sourceLen) { return bound < sourceLen ? (z_size_t)-1 : bound; } uLong ZEXPORT compressBound(uLong sourceLen) { - z_size_t bound = compressBound_z(sourceLen); - return (uLong)bound != bound ? (uLong)-1 : (uLong)bound; + uLong complen = DEFLATE_BOUND_COMPLEN(sourceLen); + + if (complen > 0) + /* Architecture-specific code provided an upper bound. */ + return complen + ZLIB_WRAPLEN; + + return sourceLen + (sourceLen >> 12) + (sourceLen >> 14) + + (sourceLen >> 25) + 13; } diff --git a/configure b/configure index 4e6c8d85b8..856e569b1b 100755 --- a/configure +++ b/configure @@ -96,6 +96,7 @@ undefined=0 insecure=0 unknown=0 enable_crcvx=1 +enable_dfltcc=0 old_cc="$CC" old_cflags="$CFLAGS" OBJC='$(OBJZ) $(OBJG)' @@ -124,6 +125,7 @@ case "$1" in echo ' [--insecure] [--static] [--64] [--libdir=LIBDIR] [--sharedlibdir=LIBDIR]' | tee -a configure.log echo ' [--includedir=INCLUDEDIR] [--mandir=MANDIR]' | tee -a configure.log echo ' [--archs="-arch i386 -arch x86_64"] [--disable-crcvx]' | tee -a configure.log + echo ' [--dfltcc] [--dfltcc-level-mask=MASK]' | tee -a configure.log exit 0 ;; -p*=* | --prefix=*) prefix=`echo $1 | sed 's/[^=]*=//'`; shift ;; -e*=* | --eprefix=*) exec_prefix=`echo $1 | sed 's/[^=]*=//'`; shift ;; @@ -158,6 +160,10 @@ case "$1" in --undefined) undefined=1; shift ;; --insecure) insecure=1; shift ;; --disable-crcvx) enable_crcvx=0; shift ;; + --dfltcc)enable_dfltcc=1; shift ;; + --dfltcc-level-mask=*) + CFLAGS="$CFLAGS -DDFLTCC_LEVEL_MASK=`echo $1 | sed 's/.*=//'`" + shift ;; *) unknown=1; echo "unknown option ignored: $1" | tee -a configure.log; shift;; esac done @@ -979,6 +985,17 @@ EOF fi fi +# enable ibm s390x dfltcc extension +HAVE_S390X_DFLTCC=0 +if test $HAVE_S390X -eq 1 && test $enable_dfltcc -eq 1; then + HAVE_S390X_DFLTCC=1 + echo "Enabeling s390x dfltcc extension ... Yes." | tee -a configure.log + CFLAGS="$CFLAGS -DHAVE_S390X_DFLTCC" + SFLAGS="$SFLAGS -DHAVE_S390X_DFLTCC" + OBJC="$OBJC dfltcc.o" + PIC_OBJC="$PIC_OBJC dfltcc.lo" +fi + # show the results in the log echo >> configure.log echo ALL = $ALL >> configure.log @@ -1012,6 +1029,7 @@ echo sharedlibdir = $sharedlibdir >> configure.log echo uname = $uname >> configure.log echo HAVE_S390X = $HAVE_S390X >> configure.log echo HAVE_S390X_VX = $HAVE_S390X_VX >> configure.log +echo HAVE_S390X_DFLTCC = $HAVE_S390X_DFLTCC >> configure.log echo VGFMAFLAG = $VGFMAFLAG >> configure.log # update Makefile with the configure results diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 19e3a567e4..e8bb67daf9 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -48,6 +48,7 @@ zlib_add_contrib_feature("GVMAT64" zlib_add_contrib_feature(INFBACK9 "with support for method 9 deflate" infback9) zlib_add_contrib_feature(CRC32VX "with S390X-CRC32VX implementation" crc32vx) +zlib_add_contrib_feature(DFLTCC "with S390X-DFLTCC deflate acceleration" dfltcc) zlib_add_contrib_lib(ADA "Ada bindings" ada) zlib_add_contrib_lib(BLAST "blast binary" blast) zlib_add_contrib_lib(IOSTREAM3 "IOStream C++ bindings V3" iostream3) diff --git a/contrib/README.contrib b/contrib/README.contrib index 9f133982d2..5d0c59478f 100644 --- a/contrib/README.contrib +++ b/contrib/README.contrib @@ -49,6 +49,10 @@ puff/ by Mark Adler crc32vx/ by Ilya Leoshkevich Hardware-accelerated CRC32 on IBM Z with Z13 VX extension. +dfltcc/ by Ilya Leoshkevich + Hardware-accelerated deflate on IBM Z with Z15 DEFLATE CONVERSION CALL + instruction. + testzlib/ by Gilles Vollant Example of the use of zlib diff --git a/contrib/dfltcc/CMakeLists.txt b/contrib/dfltcc/CMakeLists.txt new file mode 100644 index 0000000000..6e90184b50 --- /dev/null +++ b/contrib/dfltcc/CMakeLists.txt @@ -0,0 +1,63 @@ +# check if we compile for IBM s390x +# +CHECK_C_SOURCE_COMPILES(" +#ifndef __s390x__ + #error +#endif +int main() {return 0;} +" HAS_S390X_SUPPORT) + +# +# Check for IBM S390X - DFLTCC extensions +# +if(ZLIB_WITH_DFLTCC AND HAS_S390X_SUPPORT) + # check if we have static_assert + check_c_source_compiles(" + #include + static_assert(1==1,\"true\"); + " HAS_STATIC_ASSERT) + + # check if we have secure_getenv + set(CMAKE_REQUIRED_FLAGS -D_GNU_SOURCE=1) + check_c_source_compiles(" + #include + int main() { char* _foo = secure_getenv(\"PWD\");return 0; } + " HAS_SECURE_GETENV) + unset(CMAKE_REQUIRED_FLAGS) + + # check for specific headers + check_include_file(sys/sdt.h HAS_SYS_SDT_H) + + set(definitions "-DHAVE_S390X_DFLTCC=1") + if(HAS_STATIC_ASSERT) + list(APPEND definitions "-DHAVE_STATIC_ASSERT=1") + endif() + if(HAS_SECURE_GETENV) + list(APPEND definitions "-DHAVE_SECURE_GETENV=1") + list(APPEND definitions "-D_GNU_SOURCE=1") + endif() + if(HAS_SYS_SDT_H) + list(APPEND definitions "-DHAVE_SYS_SDT_H=1") + endif() + + # prepare compiling for s390x + if(ZLIB_BUILD_SHARED) + target_sources(zlib + PRIVATE + dfltcc.c + dfltcc_hooks.h + dfltcc_common.h + hooks.h) + target_compile_definitions(zlib PUBLIC ${definitions}) + endif(ZLIB_BUILD_SHARED) + if(ZLIB_BUILD_STATIC) + target_sources(zlibstatic + PRIVATE + dfltcc.c + dfltcc_hooks.h + dfltcc_common.h + hooks.h) + target_compile_definitions(zlibstatic PUBLIC ${definitions}) + endif(ZLIB_BUILD_STATIC) + +endif(ZLIB_WITH_DFLTCC AND HAS_S390X_SUPPORT) diff --git a/contrib/dfltcc/README b/contrib/dfltcc/README new file mode 100644 index 0000000000..812b0c032c --- /dev/null +++ b/contrib/dfltcc/README @@ -0,0 +1,28 @@ +IBM Z mainframes starting from version z13 provide vector instructions, which +allows vectorization of crc32. This extension is build by default when targeting +ibm s390x. However this extension can disabled if desired: + + # for configure build + $ ./configure --disable-crcvx + + # for cmake build + $ cmake .. -DZLIB_CRC32VX=off + + +IBM Z mainframes starting from version z15 provide DFLTCC instruction, +which implements deflate algorithm in hardware with estimated +compression and decompression performance orders of magnitude faster +than the current zlib and ratio comparable with that of level 1. + +This directory adds DFLTCC support. In order to enable it, the following +build commands should be used: + + $ ./configure --dfltcc + $ make + +When built like this, zlib would compress in hardware on level 1, and in +software on all other levels. Decompression will always happen in +hardware. In order to enable DFLTCC compression for levels 1-6 (i.e. to +make it used by default) one could either configure with +--dfltcc-level-mask=0x7e or set the environment variable +DFLTCC_LEVEL_MASK to 0x7e at run time. diff --git a/contrib/dfltcc/dfltcc.c b/contrib/dfltcc/dfltcc.c new file mode 100644 index 0000000000..8975460bee --- /dev/null +++ b/contrib/dfltcc/dfltcc.c @@ -0,0 +1,1013 @@ +/* dfltcc.c - SystemZ DEFLATE CONVERSION CALL support. */ + +/* + Use the following commands to build zlib with DFLTCC support: + + $ ./configure --dfltcc + $ make +*/ + +#include +#include +#include +#include +#include +#include +#include "../../zutil.h" +#include "../../deflate.h" +#include "../../inftrees.h" +#include "../../inflate.h" +#include "dfltcc.h" +#include "dfltcc_common.h" +#ifdef HAVE_SYS_SDT_H +#include +#endif +#if HAVE_STATIC_ASSERT +#include +#define z_assert_str(s) #s +#define z_static_assert(c,m) static_assert(c,z_assert_str(m)) +#else +/* + Parameter Block for Query Available Functions. + */ +#define z_static_assert(c, msg) \ + __attribute__((unused)) \ + static char static_assert_failed_ ## msg[c ? 1 : -1] +#endif +#ifdef HAVE_SECURE_GETENV +#define z_getenv(n) secure_getenv(n) +#else +#define z_getenv(n) getenv(n) +#endif + +/* + C wrapper for the DEFLATE CONVERSION CALL instruction. + */ +typedef enum { + DFLTCC_CC_OK = 0, + DFLTCC_CC_OP1_TOO_SHORT = 1, + DFLTCC_CC_OP2_TOO_SHORT = 2, + DFLTCC_CC_OP2_CORRUPT = 2, + DFLTCC_CC_AGAIN = 3, +} dfltcc_cc; + +#define DFLTCC_QAF 0 +#define DFLTCC_GDHT 1 +#define DFLTCC_CMPR 2 +#define DFLTCC_XPND 4 +#define HBT_CIRCULAR (1 << 7) +#define HB_BITS 15 +#define HB_SIZE (1 << HB_BITS) +#define DFLTCC_FACILITY 151 + +local inline dfltcc_cc dfltcc(int fn, void *param, + Bytef **op1, size_t *len1, + z_const Bytef **op2, size_t *len2, + void *hist) +{ + Bytef *t2 = op1 ? *op1 : NULL; + size_t t3 = len1 ? *len1 : 0; + z_const Bytef *t4 = op2 ? *op2 : NULL; + size_t t5 = len2 ? *len2 : 0; + register int r0 __asm__("r0") = fn; + register void *r1 __asm__("r1") = param; + register Bytef *r2 __asm__("r2") = t2; + register size_t r3 __asm__("r3") = t3; + register z_const Bytef *r4 __asm__("r4") = t4; + register size_t r5 __asm__("r5") = t5; + int cc; + + __asm__ volatile( +#ifdef HAVE_SYS_SDT_H + STAP_PROBE_ASM(zlib, dfltcc_entry, + STAP_PROBE_ASM_TEMPLATE(5)) +#endif + ".insn rrf,0xb9390000,%[r2],%[r4],%[hist],0\n" +#ifdef HAVE_SYS_SDT_H + STAP_PROBE_ASM(zlib, dfltcc_exit, + STAP_PROBE_ASM_TEMPLATE(5)) +#endif + "ipm %[cc]\n" + : [r2] "+r" (r2) + , [r3] "+r" (r3) + , [r4] "+r" (r4) + , [r5] "+r" (r5) + , [cc] "=r" (cc) + : [r0] "r" (r0) + , [r1] "r" (r1) + , [hist] "r" (hist) +#ifdef HAVE_SYS_SDT_H + , STAP_PROBE_ASM_OPERANDS(5, r2, r3, r4, r5, hist) +#endif + : "cc", "memory"); + t2 = r2; t3 = r3; t4 = r4; t5 = r5; + + if (op1) + *op1 = t2; + if (len1) + *len1 = t3; + if (op2) + *op2 = t4; + if (len2) + *len2 = t5; + return (cc >> 28) & 3; +} + +/* + Parameter Block for Query Available Functions. + */ +struct dfltcc_qaf_param { + char fns[16]; + char reserved1[8]; + char fmts[2]; + char reserved2[6]; +}; + +z_static_assert(sizeof(struct dfltcc_qaf_param) == 32, + sizeof_struct_dfltcc_qaf_param_is_32); + +local inline int is_bit_set(const char *bits, int n) +{ + return bits[n / 8] & (1 << (7 - (n % 8))); +} + +local inline void clear_bit(char *bits, int n) +{ + bits[n / 8] &= ~(1 << (7 - (n % 8))); +} + +#define DFLTCC_FMT0 0 + +/* + Parameter Block for Generate Dynamic-Huffman Table, Compress and Expand. + */ +#define CVT_CRC32 0 +#define CVT_ADLER32 1 +#define HTT_FIXED 0 +#define HTT_DYNAMIC 1 + +/* + Parameter Block for Generate Dynamic-Huffman Table, Compress and Expand. + */ + struct dfltcc_param_v0 { + uint16_t pbvn; /* Parameter-Block-Version Number */ + uint8_t mvn; /* Model-Version Number */ + uint8_t ribm; /* Reserved for IBM use */ + uint32_t reserved32 : 31; + uint32_t cf : 1; /* Continuation Flag */ + uint8_t reserved64[8]; + uint32_t nt : 1; /* New Task */ + uint32_t reserved129 : 1; + uint32_t cvt : 1; /* Check Value Type */ + uint32_t reserved131 : 1; + uint32_t htt : 1; /* Huffman-Table Type */ + uint32_t bcf : 1; /* Block-Continuation Flag */ + uint32_t bcc : 1; /* Block Closing Control */ + uint32_t bhf : 1; /* Block Header Final */ + uint32_t reserved136 : 1; + uint32_t reserved137 : 1; + uint32_t dhtgc : 1; /* DHT Generation Control */ + uint32_t reserved139 : 5; + uint32_t reserved144 : 5; + uint32_t sbb : 3; /* Sub-Byte Boundary */ + uint8_t oesc; /* Operation-Ending-Supplemental Code */ + uint32_t reserved160 : 12; + uint32_t ifs : 4; /* Incomplete-Function Status */ + uint16_t ifl; /* Incomplete-Function Length */ + uint8_t reserved192[8]; + uint8_t reserved256[8]; + uint8_t reserved320[4]; + uint16_t hl; /* History Length */ + uint32_t reserved368 : 1; + uint16_t ho : 15; /* History Offset */ + uint32_t cv; /* Check Value */ + uint32_t eobs : 15; /* End-of-block Symbol */ + uint32_t reserved431: 1; + uint8_t eobl : 4; /* End-of-block Length */ + uint32_t reserved436 : 12; + uint32_t reserved448 : 4; + uint16_t cdhtl : 12; /* Compressed-Dynamic-Huffman Table + Length */ + uint8_t reserved464[6]; + uint8_t cdht[288]; /* Compressed-Dynamic-Huffman Table */ + uint8_t reserved[24]; + uint8_t ribm2[8]; /* Reserved for IBM use */ + uint8_t csb[1152]; /* Continuation-State Buffer */ +} __attribute__((aligned(8))); +z_static_assert(sizeof(struct dfltcc_param_v0) == 1536, + sizeof_struct_dfltcc_param_v0_is_1536); + +local z_const char *oesc_msg(char *buf, int oesc) +{ + if (oesc == 0x00) + return NULL; /* Successful completion */ + else { + sprintf(buf, "Operation-Ending-Supplemental Code is 0x%.2X", oesc); + return buf; + } +} + +/* + Extension of inflate_state and deflate_state. Must be doubleword-aligned. + */ + struct dfltcc_state { + struct dfltcc_param_v0 param; /* Parameter block. */ + struct dfltcc_qaf_param af; /* Available functions. */ + uLong level_mask; /* Levels on which to use DFLTCC */ + uLong block_size; /* New block each X bytes */ + uLong block_threshold; /* New block after total_in > X */ + uLong dht_threshold; /* New block only if avail_in >= X */ + char msg[64]; /* Buffer for strm->msg */ +}; + +#define ALIGN_UP(p, size) \ + (__typeof__(p))(((uintptr_t)(p) + ((size) - 1)) & ~((size) - 1)) + +#define GET_DFLTCC_STATE(state) ((struct dfltcc_state *)( \ + (char *)(state) + ALIGN_UP(sizeof(*state), 8))) + +/* + Compress. + */ +local inline int dfltcc_can_deflate_with_params(z_streamp strm, + int level, + uInt window_bits, + int strategy) +{ + deflate_state *state = (deflate_state *)strm->state; + struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state); + + /* Unsupported compression settings */ + if ((dfltcc_state->level_mask & (1 << level)) == 0) + return 0; + if (window_bits != HB_BITS) + return 0; + if (strategy != Z_FIXED && strategy != Z_DEFAULT_STRATEGY) + return 0; + + /* Unsupported hardware */ + if (!is_bit_set(dfltcc_state->af.fns, DFLTCC_GDHT) || + !is_bit_set(dfltcc_state->af.fns, DFLTCC_CMPR) || + !is_bit_set(dfltcc_state->af.fmts, DFLTCC_FMT0)) + return 0; + + return 1; +} + +int ZLIB_INTERNAL dfltcc_can_deflate(z_streamp strm) +{ + deflate_state *state = (deflate_state *)strm->state; + + return dfltcc_can_deflate_with_params(strm, + state->level, + state->w_bits, + state->strategy); +} + +local void dfltcc_gdht(z_streamp strm) +{ + deflate_state *state = (deflate_state *)strm->state; + struct dfltcc_param_v0 *param = &GET_DFLTCC_STATE(state)->param; + size_t avail_in = avail_in = strm->avail_in; + + dfltcc(DFLTCC_GDHT, + param, NULL, NULL, + &strm->next_in, &avail_in, NULL); +} + +local dfltcc_cc dfltcc_cmpr(z_streamp strm) +{ + deflate_state *state = (deflate_state *)strm->state; + struct dfltcc_param_v0 *param = &GET_DFLTCC_STATE(state)->param; + size_t avail_in = strm->avail_in; + size_t avail_out = strm->avail_out; + dfltcc_cc cc; + + cc = dfltcc(DFLTCC_CMPR | HBT_CIRCULAR, + param, &strm->next_out, &avail_out, + &strm->next_in, &avail_in, state->window); + strm->total_in += (strm->avail_in - avail_in); + strm->total_out += (strm->avail_out - avail_out); + strm->avail_in = avail_in; + strm->avail_out = avail_out; + return cc; +} + +local void send_eobs(z_streamp strm, + z_const struct dfltcc_param_v0 *param) +{ + deflate_state *state = (deflate_state *)strm->state; + + _tr_send_bits( + state, + bi_reverse(param->eobs >> (15 - param->eobl), param->eobl), + param->eobl); + flush_pending(strm); + if (state->pending != 0) { + /* The remaining data is located in pending_out[0:pending]. If someone + * calls put_byte() - this might happen in deflate() - the byte will be + * placed into pending_buf[pending], which is incorrect. Move the + * remaining data to the beginning of pending_buf so that put_byte() is + * usable again. + */ + memmove(state->pending_buf, state->pending_out, state->pending); + state->pending_out = state->pending_buf; + } +#ifdef ZLIB_DEBUG + state->compressed_len += param->eobl; +#endif +} + +int ZLIB_INTERNAL dfltcc_deflate(z_streamp strm, int flush, + block_state *result) +{ + deflate_state *state = (deflate_state *)strm->state; + struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state); + struct dfltcc_param_v0 *param = &dfltcc_state->param; + uInt masked_avail_in; + dfltcc_cc cc; + int need_empty_block; + int soft_bcc; + int no_flush; + + if (!dfltcc_can_deflate(strm)) { + /* Clear history. */ + if (flush == Z_FULL_FLUSH) + param->hl = 0; + return 0; + } + +again: + masked_avail_in = 0; + soft_bcc = 0; + no_flush = flush == Z_NO_FLUSH; + + /* No input data. Return, except when Continuation Flag is set, which means + * that DFLTCC has buffered some output in the parameter block and needs to + * be called again in order to flush it. + */ + if (strm->avail_in == 0 && !param->cf) { + /* A block is still open, and the hardware does not support closing + * blocks without adding data. Thus, close it manually. + */ + if (!no_flush && param->bcf) { + send_eobs(strm, param); + param->bcf = 0; + } + /* Let one of deflate_* functions write a trailing empty block. */ + if (flush == Z_FINISH) + return 0; + /* Clear history. */ + if (flush == Z_FULL_FLUSH) + param->hl = 0; + /* Trigger block post-processing if necessary. */ + *result = no_flush ? need_more : block_done; + return 1; + } + + /* There is an open non-BFINAL block, we are not going to close it just + * yet, we have compressed more than DFLTCC_BLOCK_SIZE bytes and we see + * more than DFLTCC_DHT_MIN_SAMPLE_SIZE bytes. Open a new block with a new + * DHT in order to adapt to a possibly changed input data distribution. + */ + if (param->bcf && no_flush && + strm->total_in > dfltcc_state->block_threshold && + strm->avail_in >= dfltcc_state->dht_threshold) { + if (param->cf) { + /* We need to flush the DFLTCC buffer before writing the + * End-of-block Symbol. Mask the input data and proceed as usual. + */ + masked_avail_in += strm->avail_in; + strm->avail_in = 0; + no_flush = 0; + } else { + /* DFLTCC buffer is empty, so we can manually write the + * End-of-block Symbol right away. + */ + send_eobs(strm, param); + param->bcf = 0; + dfltcc_state->block_threshold = + strm->total_in + dfltcc_state->block_size; + } + } + + /* No space for compressed data. If we proceed, dfltcc_cmpr() will return + * DFLTCC_CC_OP1_TOO_SHORT without buffering header bits, but we will still + * set BCF=1, which is wrong. Avoid complications and return early. + */ + if (strm->avail_out == 0) { + *result = need_more; + return 1; + } + + /* The caller gave us too much data. Pass only one block worth of + * uncompressed data to DFLTCC and mask the rest, so that on the next + * iteration we start a new block. + */ + if (no_flush && strm->avail_in > dfltcc_state->block_size) { + masked_avail_in += (strm->avail_in - dfltcc_state->block_size); + strm->avail_in = dfltcc_state->block_size; + } + + /* When we have an open non-BFINAL deflate block and caller indicates that + * the stream is ending, we need to close an open deflate block and open a + * BFINAL one. + */ + need_empty_block = flush == Z_FINISH && param->bcf && !param->bhf; + + /* Translate stream to parameter block */ + param->cvt = state->wrap == 2 ? CVT_CRC32 : CVT_ADLER32; + if (!no_flush) + /* We need to close a block. Always do this in software - when there is + * no input data, the hardware will not honor BCC. */ + soft_bcc = 1; + if (flush == Z_FINISH && !param->bcf) + /* We are about to open a BFINAL block, set Block Header Final bit + * until the stream ends. + */ + param->bhf = 1; + /* DFLTCC-CMPR will write to next_out, so make sure that buffers with + * higher precedence are empty. + */ + Assert(state->pending == 0, "There must be no pending bytes"); + Assert(state->bi_valid < 8, "There must be less than 8 pending bits"); + param->sbb = (unsigned int)state->bi_valid; + if (param->sbb > 0) + *strm->next_out = (Bytef)state->bi_buf; + /* Honor history and check value */ + param->nt = 0; + if (state->wrap == 1) + param->cv = strm->adler; + else if (state->wrap == 2) + param->cv = ZSWAP32(strm->adler); + + /* When opening a block, choose a Huffman-Table Type */ + if (!param->bcf) { + if (state->strategy == Z_FIXED || + (strm->total_in == 0 && dfltcc_state->block_threshold > 0)) + param->htt = HTT_FIXED; + else { + param->htt = HTT_DYNAMIC; + dfltcc_gdht(strm); + } + } + + /* Deflate */ + do { + cc = dfltcc_cmpr(strm); + if (strm->avail_in < 4096 && masked_avail_in > 0) + /* We are about to call DFLTCC with a small input buffer, which is + * inefficient. Since there is masked data, there will be at least + * one more DFLTCC call, so skip the current one and make the next + * one handle more data. + */ + break; + } while (cc == DFLTCC_CC_AGAIN); + + /* Translate parameter block to stream */ + strm->msg = oesc_msg(dfltcc_state->msg, param->oesc); + state->bi_valid = param->sbb; + if (state->bi_valid == 0) + state->bi_buf = 0; /* Avoid accessing next_out */ + else + state->bi_buf = *strm->next_out & ((1 << state->bi_valid) - 1); + if (state->wrap == 1) + strm->adler = param->cv; + else if (state->wrap == 2) + strm->adler = ZSWAP32(param->cv); + + /* Unmask the input data */ + strm->avail_in += masked_avail_in; + masked_avail_in = 0; + + /* If we encounter an error, it means there is a bug in DFLTCC call */ + Assert(cc != DFLTCC_CC_OP2_CORRUPT || param->oesc == 0, "BUG"); + + /* Update Block-Continuation Flag. It will be used to check whether to call + * GDHT the next time. + */ + if (cc == DFLTCC_CC_OK) { + if (soft_bcc) { + send_eobs(strm, param); + param->bcf = 0; + dfltcc_state->block_threshold = + strm->total_in + dfltcc_state->block_size; + } else + param->bcf = 1; + if (flush == Z_FINISH) { + if (need_empty_block) + /* Make the current deflate() call also close the stream */ + return 0; + else { + bi_windup(state); + *result = finish_done; + } + } else { + if (flush == Z_FULL_FLUSH) + param->hl = 0; /* Clear history */ + *result = flush == Z_NO_FLUSH ? need_more : block_done; + } + } else { + param->bcf = 1; + *result = need_more; + } + if (strm->avail_in != 0 && strm->avail_out != 0) + goto again; /* deflate() must use all input or all output */ + return 1; +} + +/* + Expand. + */ +int ZLIB_INTERNAL dfltcc_can_inflate(z_streamp strm) +{ + struct inflate_state *state = (struct inflate_state *)strm->state; + struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state); + + /* Unsupported hardware */ + return is_bit_set(dfltcc_state->af.fns, DFLTCC_XPND) && + is_bit_set(dfltcc_state->af.fmts, DFLTCC_FMT0); +} + +local dfltcc_cc dfltcc_xpnd(z_streamp strm) +{ + struct inflate_state *state = (struct inflate_state *)strm->state; + struct dfltcc_param_v0 *param = &GET_DFLTCC_STATE(state)->param; + size_t avail_in = strm->avail_in; + size_t avail_out = strm->avail_out; + dfltcc_cc cc; + + cc = dfltcc(DFLTCC_XPND | HBT_CIRCULAR, + param, &strm->next_out, &avail_out, + &strm->next_in, &avail_in, state->window); + strm->avail_in = avail_in; + strm->avail_out = avail_out; + return cc; +} + +dfltcc_inflate_action ZLIB_INTERNAL dfltcc_inflate(z_streamp strm, int flush, + int *ret) +{ + struct inflate_state *state = (struct inflate_state *)strm->state; + struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state); + struct dfltcc_param_v0 *param = &dfltcc_state->param; + dfltcc_cc cc; + + if (flush == Z_BLOCK || flush == Z_TREES) { + /* DFLTCC does not support stopping on block boundaries */ + if (dfltcc_inflate_disable(strm)) { + *ret = Z_STREAM_ERROR; + return DFLTCC_INFLATE_BREAK; + } else + return DFLTCC_INFLATE_SOFTWARE; + } + + if (state->last) { + if (state->bits != 0) { + strm->next_in++; + strm->avail_in--; + state->bits = 0; + } + state->mode = CHECK; + return DFLTCC_INFLATE_CONTINUE; + } + + if (strm->avail_in == 0 && !param->cf) + return DFLTCC_INFLATE_BREAK; + + if (inflate_ensure_window(state)) { + state->mode = MEM; + return DFLTCC_INFLATE_CONTINUE; + } + + /* Translate stream to parameter block */ + param->cvt = ((state->wrap & 4) && state->flags) ? CVT_CRC32 : CVT_ADLER32; + param->sbb = state->bits; + if (param->hl) + param->nt = 0; /* Honor history for the first block */ + if (state->wrap & 4) + param->cv = state->flags ? ZSWAP32(state->check) : state->check; + + /* Inflate */ + do { + cc = dfltcc_xpnd(strm); + } while (cc == DFLTCC_CC_AGAIN); + + /* Translate parameter block to stream */ + strm->msg = oesc_msg(dfltcc_state->msg, param->oesc); + + if (state->wrap & 4) + strm->adler = state->check = state->flags ? ZSWAP32(param->cv) : param->cv; + state->last = cc == DFLTCC_CC_OK; + state->bits = param->sbb; + if (state->wrap & 4) + strm->adler = state->check = state->flags ? + ZSWAP32(param->cv) : param->cv; + if (cc == DFLTCC_CC_OP2_CORRUPT && param->oesc != 0) { + /* Report an error if stream is corrupted */ + state->mode = BAD; + return DFLTCC_INFLATE_CONTINUE; + } + state->mode = TYPEDO; + /* Break if operands are exhausted, otherwise continue looping */ + return (cc == DFLTCC_CC_OP1_TOO_SHORT || cc == DFLTCC_CC_OP2_TOO_SHORT) ? + DFLTCC_INFLATE_BREAK : DFLTCC_INFLATE_CONTINUE; +} + +int ZLIB_INTERNAL dfltcc_was_inflate_used(z_streamp strm) +{ + struct inflate_state *state = (struct inflate_state *)strm->state; + struct dfltcc_param_v0 *param = &GET_DFLTCC_STATE(state)->param; + + return !param->nt; +} + +/* + Rotates a circular buffer. + The implementation is based on https://cplusplus.com/reference/algorithm/rotate/ + */ +local void rotate(Bytef *start, Bytef *pivot, Bytef *end) +{ + Bytef *p = pivot; + Bytef tmp; + + while (p != start) { + tmp = *start; + *start = *p; + *p = tmp; + + start++; + p++; + + if (p == end) + p = pivot; + else if (start == pivot) + pivot = p; + } +} + +#define MIN(x, y) ({ \ + typeof(x) _x = (x); \ + typeof(y) _y = (y); \ + _x < _y ? _x : _y; \ +}) + +#define MAX(x, y) ({ \ + typeof(x) _x = (x); \ + typeof(y) _y = (y); \ + _x > _y ? _x : _y; \ +}) + +int ZLIB_INTERNAL dfltcc_inflate_disable(z_streamp strm) +{ + struct inflate_state *state = (struct inflate_state *)strm->state; + struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state); + struct dfltcc_param_v0 *param = &dfltcc_state->param; + + if (!dfltcc_can_inflate(strm)) + return 0; + if (dfltcc_was_inflate_used(strm)) + /* DFLTCC has already decompressed some data. Since there is not + * enough information to resume decompression in software, the call + * must fail. + */ + return 1; + /* DFLTCC was not used yet - decompress in software */ + memset(&dfltcc_state->af, 0, sizeof(dfltcc_state->af)); + /* Convert the window from the hardware to the software format */ + rotate(state->window, state->window + param->ho, state->window + HB_SIZE); + state->whave = state->wnext = MIN(param->hl, state->wsize); + return 0; +} + +int env_dfltcc_disabled; +int env_dfltcc_source_date_epoch; +unsigned long env_dfltcc_level_mask; +unsigned long env_dfltcc_block_size; +unsigned long env_dfltcc_block_threshold; +unsigned long env_dfltcc_dht_threshold; +unsigned long env_dfltcc_ribm; +local uint64_t cpu_facilities[(DFLTCC_FACILITY / 64) + 1]; +local struct dfltcc_qaf_param cpu_af __attribute__((aligned(8))); + +local inline int is_dfltcc_enabled(void) +{ + if (env_dfltcc_disabled) + /* User has explicitly disabled DFLTCC. */ + return 0; + + return is_bit_set((const char *)cpu_facilities, DFLTCC_FACILITY); +} + +local unsigned long xstrtoul(const char *s, unsigned long _default) +{ + char *endptr; + unsigned long result; + + if (!(s && *s)) + return _default; + errno = 0; + result = strtoul(s, &endptr, 0); + return (errno || *endptr) ? _default : result; +} + +__attribute__((constructor)) local void dfltcc_init_globals(void) +{ + const char *env; + register char r0 __asm__("r0"); + + env = z_getenv("DFLTCC"); + if (env) + env_dfltcc_disabled = !strcmp(env, "0"); + else + env_dfltcc_disabled = DFLTCC_DISABLED; + + env = z_getenv("SOURCE_DATE_EPOCH"); + if(env) + env_dfltcc_source_date_epoch = !strcmp(env, "0"); + else + env_dfltcc_source_date_epoch = DFLTCC_SOURCE_DATE_EPOCH; + + + env_dfltcc_level_mask = xstrtoul(z_getenv("DFLTCC_LEVEL_MASK"), + DFLTCC_LEVEL_MASK); + + env_dfltcc_block_size = xstrtoul(z_getenv("DFLTCC_BLOCK_SIZE"), + DFLTCC_BLOCK_SIZE); + + env_dfltcc_block_threshold = xstrtoul(z_getenv("DFLTCC_FIRST_FHT_BLOCK_SIZE"), + DFLTCC_BLOCK_THRESHOLD); + + env_dfltcc_dht_threshold = xstrtoul(z_getenv("DFLTCC_DHT_MIN_SAMPLE_SIZE"), + DFLTCC_DHT_THRESHOLD); + + env_dfltcc_ribm = xstrtoul(z_getenv("DFLTCC_RIBM"), DFLTCC_RIBM); + + memset(cpu_facilities, 0, sizeof(cpu_facilities)); + r0 = sizeof(cpu_facilities) / sizeof(cpu_facilities[0]) - 1; + /* STFLE is supported since z9-109 and only in z/Architecture mode. When + * compiling with -m31, gcc defaults to ESA mode, however, since the kernel + * is 64-bit, it's always z/Architecture mode at runtime. + */ + __asm__ volatile( +#ifndef __clang__ + ".machinemode push\n" + ".machinemode zarch\n" +#endif + "stfle %[facilities]\n" +#ifndef __clang__ + ".machinemode pop\n" +#endif + : [facilities] "=Q" (cpu_facilities) + , [r0] "+r" (r0) + : + : "cc"); + + /* Initialize available functions */ + if (is_dfltcc_enabled()) + dfltcc(DFLTCC_QAF, &cpu_af, NULL, NULL, NULL, NULL, NULL); + else + memset(&cpu_af, 0, sizeof(cpu_af)); +} + +/* + Memory management. + + DFLTCC requires parameter blocks and window to be aligned. zlib allows + users to specify their own allocation functions, so using e.g. + `posix_memalign' is not an option. Thus, we overallocate and take the + aligned portion of the buffer. +*/ +void ZLIB_INTERNAL dfltcc_reset(z_streamp strm, uInt size) +{ + struct dfltcc_state *dfltcc_state = + (struct dfltcc_state *)((char *)strm->state + ALIGN_UP(size, 8)); + + memcpy(&dfltcc_state->af, &cpu_af, sizeof(dfltcc_state->af)); + + if (env_dfltcc_source_date_epoch) + /* User needs reproducible results, but the output of DFLTCC_CMPR + * depends on buffers' page offsets. + */ + clear_bit(dfltcc_state->af.fns, DFLTCC_CMPR); + + /* Initialize parameter block */ + memset(&dfltcc_state->param, 0, sizeof(dfltcc_state->param)); + dfltcc_state->param.nt = 1; + + /* Initialize tuning parameters */ + dfltcc_state->level_mask = env_dfltcc_level_mask; + dfltcc_state->block_size = env_dfltcc_block_size; + dfltcc_state->block_threshold = env_dfltcc_block_threshold; + dfltcc_state->dht_threshold = env_dfltcc_dht_threshold; + dfltcc_state->param.ribm = env_dfltcc_ribm; +} + +voidpf ZLIB_INTERNAL dfltcc_alloc_state(z_streamp strm, uInt items, uInt size) +{ + return ZALLOC(strm, + ALIGN_UP(items * size, 8) + sizeof(struct dfltcc_state), + sizeof(unsigned char)); +} + +void ZLIB_INTERNAL dfltcc_copy_state(voidpf dst, const voidpf src, uInt size) +{ + zmemcpy(dst, src, ALIGN_UP(size, 8) + sizeof(struct dfltcc_state)); +} + +static const int PAGE_ALIGN = 0x1000; + +voidpf ZLIB_INTERNAL dfltcc_alloc_window(z_streamp strm, uInt items, uInt size) +{ + voidpf p, w; + + /* To simplify freeing, we store the pointer to the allocated buffer right + * before the window. Note that DFLTCC always uses HB_SIZE bytes. + */ + p = ZALLOC(strm, sizeof(voidpf) + MAX(items * size, HB_SIZE) + PAGE_ALIGN, + sizeof(unsigned char)); + if (p == NULL) + return NULL; + w = ALIGN_UP((char *)p + sizeof(voidpf), PAGE_ALIGN); + *(voidpf *)((char *)w - sizeof(voidpf)) = p; + return w; +} + +void ZLIB_INTERNAL dfltcc_copy_window(void *dest, const void *src, size_t n) +{ + memcpy(dest, src, MAX(n, HB_SIZE)); +} + +void ZLIB_INTERNAL dfltcc_free_window(z_streamp strm, voidpf w) +{ + if (w) + ZFREE(strm, *(voidpf *)((unsigned char *)w - sizeof(voidpf))); +} + +/* + Switching between hardware and software compression. + + DFLTCC does not support all zlib settings, e.g. generation of non-compressed + blocks or alternative window sizes. When such settings are applied on the + fly with deflateParams, we need to convert between hardware and software + window formats. +*/ +int ZLIB_INTERNAL dfltcc_deflate_params(z_streamp strm, int level, + int strategy, int *flush) +{ + deflate_state *state = (deflate_state *)strm->state; + struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state); + struct dfltcc_param_v0 *param = &dfltcc_state->param; + int could_deflate = dfltcc_can_deflate(strm); + int can_deflate = dfltcc_can_deflate_with_params(strm, + level, + state->w_bits, + strategy); + + if (can_deflate == could_deflate) + /* We continue to work in the same mode - no changes needed */ + return Z_OK; + + if (strm->total_in == 0 && param->nt == 1 && param->hl == 0) + /* DFLTCC was not used yet - no changes needed */ + return Z_OK; + + /* For now, do not convert between window formats - simply get rid of the + * old data instead. + */ + *flush = Z_FULL_FLUSH; + return Z_OK; +} + +int ZLIB_INTERNAL dfltcc_deflate_done(z_streamp strm, int flush) +{ + deflate_state *state = (deflate_state *)strm->state; + struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state); + struct dfltcc_param_v0 *param = &dfltcc_state->param; + + /* When deflate(Z_FULL_FLUSH) is called with small avail_out, it might + * close the block without resetting the compression state. Detect this + * situation and return that deflation is not done. + */ + if (flush == Z_FULL_FLUSH && strm->avail_out == 0) + return 0; + + /* Return that deflation is not done if DFLTCC is used and either it + * buffered some data (Continuation Flag is set), or has not written EOBS + * yet (Block-Continuation Flag is set). + */ + return !dfltcc_can_deflate(strm) || (!param->cf && !param->bcf); +} + +/* + Preloading history. +*/ +local void append_history(struct dfltcc_param_v0 *param, + Bytef *history, + const Bytef *buf, + uInt count) +{ + size_t offset; + size_t n; + + /* Do not use more than 32K */ + if (count > HB_SIZE) { + buf += count - HB_SIZE; + count = HB_SIZE; + } + offset = (param->ho + param->hl) % HB_SIZE; + if (offset + count <= HB_SIZE) + /* Circular history buffer does not wrap - copy one chunk */ + zmemcpy(history + offset, buf, count); + else { + /* Circular history buffer wraps - copy two chunks */ + n = HB_SIZE - offset; + zmemcpy(history + offset, buf, n); + zmemcpy(history, buf + n, count - n); + } + n = param->hl + count; + if (n <= HB_SIZE) + /* All history fits into buffer - no need to discard anything */ + param->hl = n; + else { + /* History does not fit into buffer - discard extra bytes */ + param->ho = (param->ho + (n - HB_SIZE)) % HB_SIZE; + param->hl = HB_SIZE; + } +} + +local void get_history(struct dfltcc_param_v0 *param, + const Bytef *history, + Bytef *buf) +{ + if (param->ho + param->hl <= (uint16_t) HB_SIZE) + /* Circular history buffer does not wrap - copy one chunk */ + memcpy(buf, history + param->ho, param->hl); + else { + /* Circular history buffer wraps - copy two chunks */ + memcpy(buf, history + param->ho, HB_SIZE - param->ho); + memcpy(buf + HB_SIZE - param->ho, history, param->ho + param->hl - HB_SIZE); + } +} + +int ZLIB_INTERNAL dfltcc_deflate_set_dictionary(z_streamp strm, + const Bytef *dictionary, + uInt dict_length) +{ + deflate_state *state = (deflate_state *)strm->state; + struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state); + struct dfltcc_param_v0 *param = &dfltcc_state->param; + + append_history(param, state->window, dictionary, dict_length); + state->strstart = 1; /* Add FDICT to zlib header */ + state->block_start = state->strstart; /* Make deflate_stored happy */ + return Z_OK; +} + +int ZLIB_INTERNAL dfltcc_deflate_get_dictionary(z_streamp strm, + Bytef *dictionary, + uInt *dict_length) +{ + deflate_state *state = (deflate_state *)strm->state; + struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state); + struct dfltcc_param_v0 *param = &dfltcc_state->param; + + if (dictionary) + get_history(param, state->window, dictionary); + if (dict_length) + *dict_length = param->hl; + return Z_OK; +} + +int ZLIB_INTERNAL dfltcc_inflate_set_dictionary(z_streamp strm, + const Bytef *dictionary, + uInt dict_length) +{ + struct inflate_state *state = (struct inflate_state *)strm->state; + struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state); + struct dfltcc_param_v0 *param = &dfltcc_state->param; + + if (inflate_ensure_window(state)) { + state->mode = MEM; + return Z_MEM_ERROR; + } + + append_history(param, state->window, dictionary, dict_length); + state->havedict = 1; + return Z_OK; +} + +int ZLIB_INTERNAL dfltcc_inflate_get_dictionary(z_streamp strm, + Bytef *dictionary, + uInt *dict_length) +{ + struct inflate_state *state = (struct inflate_state *)strm->state; + struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state); + struct dfltcc_param_v0 *param = &dfltcc_state->param; + + if (dictionary && state->window) + get_history(param, state->window, dictionary); + if (dict_length) + *dict_length = param->hl; + return Z_OK; +} diff --git a/contrib/dfltcc/dfltcc.h b/contrib/dfltcc/dfltcc.h new file mode 100644 index 0000000000..ca2349f011 --- /dev/null +++ b/contrib/dfltcc/dfltcc.h @@ -0,0 +1,86 @@ +#ifndef DFLTCC_H +#define DFLTCC_H + +#include "../../deflate.h" + +#include + +/* + Sizes of deflate block parts. + */ +#define DFLTCC_BLOCK_HEADER_BITS 3 +#define DFLTCC_HLITS_COUNT_BITS 5 +#define DFLTCC_HDISTS_COUNT_BITS 5 +#define DFLTCC_HCLENS_COUNT_BITS 4 +#define DFLTCC_MAX_HCLENS 19 +#define DFLTCC_HCLEN_BITS 3 +#define DFLTCC_MAX_HLITS 286 +#define DFLTCC_MAX_HDISTS 30 +#define DFLTCC_MAX_HLIT_HDIST_BITS 7 +#define DFLTCC_MAX_SYMBOL_BITS 16 +#define DFLTCC_MAX_EOBS_BITS 15 +#define DFLTCC_MAX_PADDING_BITS 7 + +#define DEFLATE_BOUND_COMPLEN(source_len) \ + ((DFLTCC_BLOCK_HEADER_BITS + \ + DFLTCC_HLITS_COUNT_BITS + \ + DFLTCC_HDISTS_COUNT_BITS + \ + DFLTCC_HCLENS_COUNT_BITS + \ + DFLTCC_MAX_HCLENS * DFLTCC_HCLEN_BITS + \ + (DFLTCC_MAX_HLITS + DFLTCC_MAX_HDISTS) * DFLTCC_MAX_HLIT_HDIST_BITS + \ + (source_len) * DFLTCC_MAX_SYMBOL_BITS + \ + DFLTCC_MAX_EOBS_BITS + \ + DFLTCC_MAX_PADDING_BITS) >> 3) + +voidpf ZLIB_INTERNAL dfltcc_alloc_state(z_streamp strm, uInt items, uInt size); +void ZLIB_INTERNAL dfltcc_copy_state(voidpf dst, const voidpf src, uInt size); +void ZLIB_INTERNAL dfltcc_reset(z_streamp strm, uInt size); +voidpf ZLIB_INTERNAL dfltcc_alloc_window(z_streamp strm, uInt items, + uInt size); +void ZLIB_INTERNAL dfltcc_copy_window(void *dest, const void *src, size_t n); +void ZLIB_INTERNAL dfltcc_free_window(z_streamp strm, voidpf w); +#define DFLTCC_BLOCK_HEADER_BITS 3 +#define DFLTCC_HLITS_COUNT_BITS 5 +#define DFLTCC_HDISTS_COUNT_BITS 5 +#define DFLTCC_HCLENS_COUNT_BITS 4 +#define DFLTCC_MAX_HCLENS 19 +#define DFLTCC_HCLEN_BITS 3 +#define DFLTCC_MAX_HLITS 286 +#define DFLTCC_MAX_HDISTS 30 +#define DFLTCC_MAX_HLIT_HDIST_BITS 7 +#define DFLTCC_MAX_SYMBOL_BITS 16 +#define DFLTCC_MAX_EOBS_BITS 15 +#define DFLTCC_MAX_PADDING_BITS 7 + +int ZLIB_INTERNAL dfltcc_can_inflate(z_streamp strm); +typedef enum { + DFLTCC_INFLATE_CONTINUE, + DFLTCC_INFLATE_BREAK, + DFLTCC_INFLATE_SOFTWARE, +} dfltcc_inflate_action; +dfltcc_inflate_action ZLIB_INTERNAL dfltcc_inflate(z_streamp strm, + int flush, int *ret); +int ZLIB_INTERNAL dfltcc_was_inflate_used(z_streamp strm); +int ZLIB_INTERNAL dfltcc_inflate_disable(z_streamp strm); +int ZLIB_INTERNAL dfltcc_inflate_set_dictionary(z_streamp strm, + const Bytef *dictionary, + uInt dict_length); +int ZLIB_INTERNAL dfltcc_inflate_get_dictionary(z_streamp strm, + Bytef *dictionary, + uInt* dict_length); + +int ZLIB_INTERNAL dfltcc_can_deflate(z_streamp strm); +int ZLIB_INTERNAL dfltcc_deflate(z_streamp strm, + int flush, + block_state *result); +int ZLIB_INTERNAL dfltcc_deflate_params(z_streamp strm, int level, + int strategy, int *flush); +int ZLIB_INTERNAL dfltcc_deflate_done(z_streamp strm, int flush); +int ZLIB_INTERNAL dfltcc_deflate_set_dictionary(z_streamp strm, + const Bytef *dictionary, + uInt dict_length); +int ZLIB_INTERNAL dfltcc_deflate_get_dictionary(z_streamp strm, + Bytef *dictionary, + uInt* dict_length); + +#endif diff --git a/contrib/dfltcc/dfltcc_common.h b/contrib/dfltcc/dfltcc_common.h new file mode 100644 index 0000000000..aec7c1e17f --- /dev/null +++ b/contrib/dfltcc/dfltcc_common.h @@ -0,0 +1,79 @@ +#ifndef Z_DFLTCC_COMMON_H +#define Z_DFLTCC_COMMON_H + +/* + Runtime Tuning parameters. +*/ + +/* + control usage of DFLTCC + + env variable: DFLTCC + 0 or unset: use DFLTCC algorithm (default) + 1: disable DFLTCC + */ +extern int env_dfltcc_disabled; +#define DFLTCC_DISABLED 0 + +/* + control usage of DFLTCC_CMPR + set this to 1 if you need reproducible builds + + env variable: SOURCE_DATE_EPOCH + 0 or unset: use DFLTCC_CMPR + 1: disable DFLTCC_CMPR + */ +extern int env_dfltcc_source_date_epoch; +#define DFLTCC_SOURCE_DATE_EPOCH 0 + +/* + disable DFLTCC for specific compression levels + + env variable: DFLTCC_LEVEL_MASK + valid range: 0-10 / 0x0-0xA + default: 0x2 - disable for compresion level 0 to 2 + */ +extern unsigned long env_dfltcc_level_mask; +#ifndef DFLTCC_LEVEL_MASK /* can be defined by configure */ +#define DFLTCC_LEVEL_MASK 0x2 +#endif +/* + New block each X bytes + + env variable: DFLTCC_BLOCK_SIZE + valid range: > 262144 / 0x40000 (256K) + default: 1048576 / 0x100000 (1M) + */ +extern unsigned long env_dfltcc_block_size; +#define DFLTCC_BLOCK_SIZE 0x100000 + +/* + New block after total_in > X + + env variable: DFLTCC_FIRST_FHT_BLOCK_SIZE + default: 4096 / 0x1000 + */ +extern unsigned long env_dfltcc_block_threshold; +#define DFLTCC_BLOCK_THRESHOLD 0x1000 + +/* + New block only if avail_in >= X + + env variable: DFLTCC_DHT_MIN_SAMPLE_SIZE + default: 4096 / 0x1000 +*/ +extern unsigned long env_dfltcc_dht_threshold; +#define DFLTCC_DHT_THRESHOLD 0x1000 + +/* + default value for DFLTCC_RIBM register + + env variable: DFLTCC_RIBM + default: 0 + valid range: 0-255 / 0x0-0xFF +*/ +extern unsigned long env_dfltcc_ribm; +#define DFLTCC_RIBM 0 + + +#endif diff --git a/contrib/dfltcc/dfltcc_hooks.h b/contrib/dfltcc/dfltcc_hooks.h new file mode 100644 index 0000000000..6cf011ef7a --- /dev/null +++ b/contrib/dfltcc/dfltcc_hooks.h @@ -0,0 +1,122 @@ +#ifndef DFLTCC_HOOKS_H +#define DFLTCC_HOOKS_H + +#include "dfltcc.h" + +/** + * DEFLATE HOOKS + */ +#define DEFLATE_BOUND_ADJUST_COMPLEN(strm, complen, source_len) \ + do { \ + if (deflateStateCheck((strm)) || dfltcc_can_deflate((strm))) \ + (complen) = DEFLATE_BOUND_COMPLEN(source_len); \ + } while (0) + +#define DEFLATE_BOUND_COMPLEN(source_len) \ + ((DFLTCC_BLOCK_HEADER_BITS + \ + DFLTCC_HLITS_COUNT_BITS + \ + DFLTCC_HDISTS_COUNT_BITS + \ + DFLTCC_HCLENS_COUNT_BITS + \ + DFLTCC_MAX_HCLENS * DFLTCC_HCLEN_BITS + \ + (DFLTCC_MAX_HLITS + DFLTCC_MAX_HDISTS) * DFLTCC_MAX_HLIT_HDIST_BITS + \ + (source_len) * DFLTCC_MAX_SYMBOL_BITS + \ + DFLTCC_MAX_EOBS_BITS + \ + DFLTCC_MAX_PADDING_BITS) >> 3) + +#define DEFLATE_DONE dfltcc_deflate_done + +#define DEFLATE_GET_DICTIONARY_HOOK(strm, dict, dict_len) \ + do { \ + if (dfltcc_can_deflate((strm))) \ + return dfltcc_deflate_get_dictionary((strm), (dict), (dict_len)); \ + } while (0) + +#define DEFLATE_HOOK dfltcc_deflate + +#define DEFLATE_NEED_CHECKSUM(strm) (!dfltcc_can_deflate((strm))) + +#define DEFLATE_NEED_CONSERVATIVE_BOUND(strm) (dfltcc_can_deflate((strm))) + +#define DEFLATE_PARAMS_HOOK(strm, level, strategy, hook_flush) \ + do { \ + int err; \ +\ + err = dfltcc_deflate_params((strm), \ + (level), \ + (strategy), \ + (hook_flush)); \ + if (err == Z_STREAM_ERROR) \ + return err; \ + } while (0) + +#define DEFLATE_RESET_KEEP_HOOK(strm) \ + dfltcc_reset((strm), sizeof(deflate_state)) + +#define DEFLATE_SET_DICTIONARY_HOOK(strm, dict, dict_len) \ + do { \ + if (dfltcc_can_deflate((strm))) \ + return dfltcc_deflate_set_dictionary((strm), (dict), (dict_len)); \ + } while (0) + + +/** + * INFLATE HOOKS + */ +#define INFLATE_GET_DICTIONARY_HOOK(strm, dict, dict_len) \ + do { \ + if (dfltcc_can_inflate(strm)) \ + return dfltcc_inflate_get_dictionary(strm, dict, dict_len); \ + } while (0) + +#define INFLATE_MARK_HOOK(strm) \ + do { \ + if (dfltcc_was_inflate_used((strm))) return -(1L << 16); \ + } while (0) + +#define INFLATE_NEED_CHECKSUM(strm) (!dfltcc_can_inflate((strm))) + +#define INFLATE_NEED_UPDATEWINDOW(strm) (!dfltcc_can_inflate((strm))) + +#define INFLATE_PRIME_HOOK(strm, bits, value) \ + do { if (dfltcc_inflate_disable((strm))) return Z_STREAM_ERROR; } while (0) + +#define INFLATE_RESET_KEEP_HOOK(strm) \ + dfltcc_reset((strm), sizeof(struct inflate_state)) + +#define INFLATE_SET_DICTIONARY_HOOK(strm, dict, dict_len) \ + do { \ + if (dfltcc_can_inflate(strm)) \ + return dfltcc_inflate_set_dictionary(strm, dict, dict_len); \ + } while (0) + +#define INFLATE_SYNC_POINT_HOOK(strm) \ + do { \ + if (dfltcc_was_inflate_used((strm))) return Z_STREAM_ERROR; \ + } while (0) + +#define INFLATE_TYPEDO_HOOK(strm, flush) \ + if (dfltcc_can_inflate((strm))) { \ + dfltcc_inflate_action action; \ + \ + RESTORE(); \ + action = dfltcc_inflate((strm), (flush), &ret); \ + LOAD(); \ + if (action == DFLTCC_INFLATE_CONTINUE) \ + break; \ + else if (action == DFLTCC_INFLATE_BREAK) \ + goto inf_leave; \ + } + + +/** + * MEMORY HOOKS + */ +#define TRY_FREE_WINDOW dfltcc_free_window +#define ZALLOC_STATE dfltcc_alloc_state +#define ZALLOC_WINDOW dfltcc_alloc_window +#define ZCOPY_STATE dfltcc_copy_state +#define ZCOPY_WINDOW dfltcc_copy_window +#define ZFREE_STATE ZFREE +#define ZFREE_WINDOW dfltcc_free_window + +#endif /* DFLTCC_HOOKS_H */ diff --git a/contrib/dfltcc/hooks.h b/contrib/dfltcc/hooks.h new file mode 100644 index 0000000000..cd4b7cc2ad --- /dev/null +++ b/contrib/dfltcc/hooks.h @@ -0,0 +1,47 @@ +/* contrib/dfltcc/hooks.h -- architecture-specific hook macro defaults for zlib + * When HAVE_S390X_DFLTCC is defined, the real implementations are provided + * by contrib/dfltcc/dfltcc_hooks.h instead. This file only defines the + * no-op defaults so that the core source files need not include any + * contrib header unconditionally. + */ + +#ifndef ZLIB_CONTRIB_HOOKS_H +#define ZLIB_CONTRIB_HOOKS_H + +#ifndef HAVE_S390X_DFLTCC + +/* Deflate hooks -- no-ops that restore original behavior */ +#define DEFLATE_BOUND_ADJUST_COMPLEN(strm, complen, sourceLen) do {} while (0) +#define DEFLATE_BOUND_COMPLEN(source_len) 0 +#define DEFLATE_DONE(strm, flush) 1 +#define DEFLATE_GET_DICTIONARY_HOOK(strm, dict, dict_len) do {} while (0) +#define DEFLATE_HOOK(strm, flush, bstate) 0 +#define DEFLATE_NEED_CHECKSUM(strm) 1 +#define DEFLATE_NEED_CONSERVATIVE_BOUND(strm) 0 +#define DEFLATE_PARAMS_HOOK(strm, level, strategy, hook_flush) do {} while (0) +#define DEFLATE_RESET_KEEP_HOOK(strm) do {} while (0) +#define DEFLATE_SET_DICTIONARY_HOOK(strm, dict, dict_len) do {} while (0) + +/* Inflate hooks -- no-ops that restore original behavior */ +#define INFLATE_GET_DICTIONARY_HOOK(strm, dict, dict_len) do {} while (0) +#define INFLATE_MARK_HOOK(strm) do {} while (0) +#define INFLATE_NEED_CHECKSUM(strm) 1 +#define INFLATE_NEED_UPDATEWINDOW(strm) 1 +#define INFLATE_PRIME_HOOK(strm, bits, value) do {} while (0) +#define INFLATE_RESET_KEEP_HOOK(strm) do {} while (0) +#define INFLATE_SET_DICTIONARY_HOOK(strm, dict, dict_len) do {} while (0) +#define INFLATE_SYNC_POINT_HOOK(strm) do {} while (0) +#define INFLATE_TYPEDO_HOOK(strm, flush) do {} while (0) + +/* Memory hooks -- map to the original allocator calls */ +#define TRY_FREE_WINDOW TRY_FREE +#define ZALLOC_STATE ZALLOC +#define ZALLOC_WINDOW ZALLOC +#define ZCOPY_STATE zmemcpy +#define ZCOPY_WINDOW zmemcpy +#define ZFREE_STATE ZFREE +#define ZFREE_WINDOW ZFREE + +#endif /* !HAVE_S390X_DFLTCC */ + +#endif /* ZLIB_CONTRIB_HOOKS_H */ diff --git a/deflate.c b/deflate.c index a5be5e6947..c601ecaf55 100644 --- a/deflate.c +++ b/deflate.c @@ -50,6 +50,11 @@ /* @(#) $Id$ */ #include "deflate.h" +#ifdef HAVE_S390X_DFLTCC +# include "contrib/dfltcc/dfltcc_hooks.h" +#else +# include "contrib/dfltcc/hooks.h" +#endif const char deflate_copyright[] = " deflate 1.3.2.1 Copyright 1995-2026 Jean-loup Gailly and Mark Adler "; @@ -60,13 +65,6 @@ const char deflate_copyright[] = copyright string in the executable of your product. */ -typedef enum { - need_more, /* block not completed, need more input or more output */ - block_done, /* block flush performed */ - finish_started, /* finish started, need only more output at next deflate */ - finish_done /* finish done, accept no more input or output */ -} block_state; - typedef block_state (*compress_func)(deflate_state *s, int flush); /* Compression function. Returns the block state after the call. */ @@ -225,6 +223,10 @@ local unsigned read_buf(z_streamp strm, Bytef *buf, unsigned size) { strm->avail_in -= len; zmemcpy(buf, strm->next_in, len); +#ifdef HAVE_S390X_DFLTCC + if (!DEFLATE_NEED_CHECKSUM(strm)) {} + else +#endif if (strm->state->wrap == 1) { strm->adler = adler32(strm->adler, buf, len); } @@ -437,7 +439,7 @@ int ZEXPORT deflateInit2_(z_streamp strm, int level, int method, return Z_STREAM_ERROR; } if (windowBits == 8) windowBits = 9; /* until 256-byte window bug fixed */ - s = (deflate_state *) ZALLOC(strm, 1, sizeof(deflate_state)); + s = (deflate_state *) ZALLOC_STATE(strm, 1, sizeof(deflate_state)); if (s == Z_NULL) return Z_MEM_ERROR; zmemzero(s, sizeof(deflate_state)); strm->state = (struct internal_state FAR *)s; @@ -455,7 +457,7 @@ int ZEXPORT deflateInit2_(z_streamp strm, int level, int method, s->hash_mask = s->hash_size - 1; s->hash_shift = ((s->hash_bits + MIN_MATCH-1) / MIN_MATCH); - s->window = (Bytef *) ZALLOC(strm, s->w_size, 2*sizeof(Byte)); + s->window = (Bytef *) ZALLOC_WINDOW(strm, s->w_size, 2*sizeof(Byte)); s->prev = (Posf *) ZALLOC(strm, s->w_size, sizeof(Pos)); s->head = (Posf *) ZALLOC(strm, s->hash_size, sizeof(Pos)); @@ -574,6 +576,7 @@ int ZEXPORT deflateSetDictionary(z_streamp strm, const Bytef *dictionary, /* when using zlib wrappers, compute Adler-32 for provided dictionary */ if (wrap == 1) strm->adler = adler32(strm->adler, dictionary, dictLength); + DEFLATE_SET_DICTIONARY_HOOK(strm, dictionary, dictLength); s->wrap = 0; /* avoid computing Adler-32 in read_buf */ /* if dictionary would fill window, just replace the history */ @@ -629,6 +632,7 @@ int ZEXPORT deflateGetDictionary(z_streamp strm, Bytef *dictionary, if (deflateStateCheck(strm)) return Z_STREAM_ERROR; + DEFLATE_GET_DICTIONARY_HOOK(strm, dictionary, dictLength); s = strm->state; len = s->strstart + s->lookahead; if (len > s->w_size) @@ -673,6 +677,8 @@ int ZEXPORT deflateResetKeep(z_streamp strm) { _tr_init(s); + DEFLATE_RESET_KEEP_HOOK(strm); + return Z_OK; } @@ -774,6 +780,7 @@ int ZEXPORT deflatePrime(z_streamp strm, int bits, int value) { int ZEXPORT deflateParams(z_streamp strm, int level, int strategy) { deflate_state *s; compress_func func; + int hook_flush = Z_NO_FLUSH; if (deflateStateCheck(strm)) return Z_STREAM_ERROR; s = strm->state; @@ -786,15 +793,18 @@ int ZEXPORT deflateParams(z_streamp strm, int level, int strategy) { if (level < 0 || level > 9 || strategy < 0 || strategy > Z_FIXED) { return Z_STREAM_ERROR; } + DEFLATE_PARAMS_HOOK(strm, level, strategy, &hook_flush); func = configuration_table[s->level].func; - if ((strategy != s->strategy || func != configuration_table[level].func) && - s->last_flush != -2) { + if (((strategy != s->strategy || func != configuration_table[level].func) && + s->last_flush != -2) || hook_flush != Z_NO_FLUSH) { /* Flush the last buffer: */ - int err = deflate(strm, Z_BLOCK); + int flush = RANK(hook_flush) > RANK(Z_BLOCK) ? hook_flush : Z_BLOCK; + int err = deflate(strm, flush); if (err == Z_STREAM_ERROR) return err; - if (strm->avail_in || (s->strstart - s->block_start) + s->lookahead) + if (strm->avail_in || (s->strstart - s->block_start) + s->lookahead || + !DEFLATE_DONE(strm, flush)) return Z_BUF_ERROR; } if (s->level != level) { @@ -862,15 +872,13 @@ z_size_t ZEXPORT deflateBound_z(z_streamp strm, z_size_t sourceLen) { ~13% overhead plus a small constant */ fixedlen = sourceLen + (sourceLen >> 3) + (sourceLen >> 8) + (sourceLen >> 9) + 4; - if (fixedlen < sourceLen) - fixedlen = (z_size_t)-1; + DEFLATE_BOUND_ADJUST_COMPLEN(strm, fixedlen, sourceLen); /* upper bound for stored blocks with length 127 (memLevel == 1) -- ~4% overhead plus a small constant */ storelen = sourceLen + (sourceLen >> 5) + (sourceLen >> 7) + (sourceLen >> 11) + 7; - if (storelen < sourceLen) - storelen = (z_size_t)-1; + DEFLATE_BOUND_ADJUST_COMPLEN(strm, storelen, sourceLen); /* if can't get parameters, return larger bound plus a wrapper */ if (deflateStateCheck(strm)) { @@ -914,11 +922,10 @@ z_size_t ZEXPORT deflateBound_z(z_streamp strm, z_size_t sourceLen) { } /* if not default parameters, return one of the conservative bounds */ - if (s->w_bits != 15 || s->hash_bits != 8 + 7) { - bound = s->w_bits <= s->hash_bits && s->level ? fixedlen : - storelen; - return bound + wraplen < bound ? (z_size_t)-1 : bound + wraplen; - } + if (DEFLATE_NEED_CONSERVATIVE_BOUND(strm) || + s->w_bits != 15 || s->hash_bits != 8 + 7) + return (s->w_bits <= s->hash_bits && s->level ? fixedlen : storelen) + + wraplen; /* default settings: return tight bound for that case -- ~0.03% overhead plus a small constant */ @@ -947,7 +954,11 @@ local void putShortMSB(deflate_state *s, uInt b) { * applications may wish to modify it to avoid allocating a large * strm->next_out buffer and copying into it. (See also read_buf()). */ +#ifdef HAVE_S390X_DFLTCC +void ZLIB_INTERNAL flush_pending(z_streamp strm) { +#else local void flush_pending(z_streamp strm) { +#endif unsigned len; deflate_state *s = strm->state; @@ -1214,7 +1225,8 @@ int ZEXPORT deflate(z_streamp strm, int flush) { (flush != Z_NO_FLUSH && s->status != FINISH_STATE)) { block_state bstate; - bstate = s->level == 0 ? deflate_stored(s, flush) : + bstate = DEFLATE_HOOK(strm, flush, &bstate) ? bstate : + s->level == 0 ? deflate_stored(s, flush) : s->strategy == Z_HUFFMAN_ONLY ? deflate_huff(s, flush) : s->strategy == Z_RLE ? deflate_rle(s, flush) : (*(configuration_table[s->level].func))(s, flush); @@ -1301,9 +1313,9 @@ int ZEXPORT deflateEnd(z_streamp strm) { TRY_FREE(strm, strm->state->pending_buf); TRY_FREE(strm, strm->state->head); TRY_FREE(strm, strm->state->prev); - TRY_FREE(strm, strm->state->window); + TRY_FREE_WINDOW(strm, strm->state->window); - ZFREE(strm, strm->state); + ZFREE_STATE(strm, strm->state); strm->state = Z_NULL; return status == BUSY_STATE ? Z_DATA_ERROR : Z_OK; @@ -1332,14 +1344,14 @@ int ZEXPORT deflateCopy(z_streamp dest, z_streamp source) { zmemcpy(dest, source, sizeof(z_stream)); - ds = (deflate_state *) ZALLOC(dest, 1, sizeof(deflate_state)); + ds = (deflate_state *) ZALLOC_STATE(dest, 1, sizeof(deflate_state)); if (ds == Z_NULL) return Z_MEM_ERROR; zmemzero(ds, sizeof(deflate_state)); dest->state = (struct internal_state FAR *) ds; - zmemcpy(ds, ss, sizeof(deflate_state)); + ZCOPY_STATE((voidpf)ds, (voidpf)ss, sizeof(deflate_state)); ds->strm = dest; - ds->window = (Bytef *) ZALLOC(dest, ds->w_size, 2*sizeof(Byte)); + ds->window = (Bytef *) ZALLOC_WINDOW(dest, ds->w_size, 2*sizeof(Byte)); ds->prev = (Posf *) ZALLOC(dest, ds->w_size, sizeof(Pos)); ds->head = (Posf *) ZALLOC(dest, ds->hash_size, sizeof(Pos)); ds->pending_buf = (uchf *) ZALLOC(dest, ds->lit_bufsize, LIT_BUFS); diff --git a/deflate.h b/deflate.h index 0732ba83ad..6c2188644a 100644 --- a/deflate.h +++ b/deflate.h @@ -316,6 +316,7 @@ void ZLIB_INTERNAL _tr_flush_bits(deflate_state *s); void ZLIB_INTERNAL _tr_align(deflate_state *s); void ZLIB_INTERNAL _tr_stored_block(deflate_state *s, charf *buf, ulg stored_len, int last); +void ZLIB_INTERNAL _tr_send_bits(deflate_state *s, int value, int length); #define d_code(dist) \ ((dist) < 256 ? _dist_code[dist] : _dist_code[256+((dist)>>7)]) @@ -380,4 +381,20 @@ void ZLIB_INTERNAL _tr_stored_block(deflate_state *s, charf *buf, flush = _tr_tally(s, distance, length) #endif +typedef enum { + need_more, /* block not completed, need more input or more output */ + block_done, /* block flush performed */ + finish_started, /* finish started, need only more output at next deflate */ + finish_done /* finish done, accept no more input or output */ +} block_state; + +/* When HAVE_S390X_DFLTCC is defined, the following functions are promoted + * from local to ZLIB_INTERNAL so that the DFLTCC code in contrib can call + * them. */ +#ifdef HAVE_S390X_DFLTCC +unsigned ZLIB_INTERNAL bi_reverse(unsigned code, int len); +void ZLIB_INTERNAL bi_windup(deflate_state *s); +void ZLIB_INTERNAL flush_pending(z_streamp strm); +#endif /* HAVE_S390X_DFLTCC */ + #endif /* DEFLATE_H */ diff --git a/inflate.c b/inflate.c index 5f5d4922b7..860022a62b 100644 --- a/inflate.c +++ b/inflate.c @@ -84,6 +84,11 @@ #include "inftrees.h" #include "inflate.h" #include "inffast.h" +#ifdef HAVE_S390X_DFLTCC +# include "contrib/dfltcc/dfltcc_hooks.h" +#else +# include "contrib/dfltcc/hooks.h" +#endif local int inflateStateCheck(z_streamp strm) { struct inflate_state FAR *state; @@ -118,6 +123,7 @@ int ZEXPORT inflateResetKeep(z_streamp strm) { state->lencode = state->distcode = state->next = state->codes; state->sane = 1; state->back = -1; + INFLATE_RESET_KEEP_HOOK(strm); Tracev((stderr, "inflate: reset\n")); return Z_OK; } @@ -160,7 +166,7 @@ int ZEXPORT inflateReset2(z_streamp strm, int windowBits) { if (windowBits && (windowBits < 8 || windowBits > 15)) return Z_STREAM_ERROR; if (state->window != Z_NULL && state->wbits != (unsigned)windowBits) { - ZFREE(strm, state->window); + ZFREE_WINDOW(strm, state->window); state->window = Z_NULL; } @@ -195,7 +201,7 @@ int ZEXPORT inflateInit2_(z_streamp strm, int windowBits, strm->zfree = zcfree; #endif state = (struct inflate_state FAR *) - ZALLOC(strm, 1, sizeof(struct inflate_state)); + ZALLOC_STATE(strm, 1, sizeof(struct inflate_state)); if (state == Z_NULL) return Z_MEM_ERROR; zmemzero(state, sizeof(struct inflate_state)); Tracev((stderr, "inflate: allocated\n")); @@ -205,7 +211,7 @@ int ZEXPORT inflateInit2_(z_streamp strm, int windowBits, state->mode = HEAD; /* to pass state test in inflateReset2() */ ret = inflateReset2(strm, windowBits); if (ret != Z_OK) { - ZFREE(strm, state); + ZFREE_STATE(strm, state); strm->state = Z_NULL; } return ret; @@ -222,6 +228,7 @@ int ZEXPORT inflatePrime(z_streamp strm, int bits, int value) { if (inflateStateCheck(strm)) return Z_STREAM_ERROR; if (bits == 0) return Z_OK; + INFLATE_PRIME_HOOK(strm, bits, value); state = (struct inflate_state FAR *)strm->state; if (bits < 0) { state->hold = 0; @@ -235,6 +242,30 @@ int ZEXPORT inflatePrime(z_streamp strm, int bits, int value) { return Z_OK; } +#ifdef HAVE_S390X_DFLTCC +int ZLIB_INTERNAL inflate_ensure_window(struct inflate_state *state) +#else +local int inflate_ensure_window(struct inflate_state *state) +#endif +{ + /* if it hasn't been done already, allocate space for the window */ + if (state->window == Z_NULL) { + state->window = (unsigned char FAR *) + ZALLOC_WINDOW(state->strm, 1U << state->wbits, + sizeof(unsigned char)); + if (state->window == Z_NULL) return 1; + } + + /* if window not in use yet, initialize */ + if (state->wsize == 0) { + state->wsize = 1U << state->wbits; + state->wnext = 0; + state->whave = 0; + } + + return 0; +} + /* Update the window with the last wsize (normally 32K) bytes written before returning. If window does not exist yet, create it. This is only called @@ -255,20 +286,7 @@ local int updatewindow(z_streamp strm, const Bytef *end, unsigned copy) { state = (struct inflate_state FAR *)strm->state; - /* if it hasn't been done already, allocate space for the window */ - if (state->window == Z_NULL) { - state->window = (unsigned char FAR *) - ZALLOC(strm, 1U << state->wbits, - sizeof(unsigned char)); - if (state->window == Z_NULL) return 1; - } - - /* if window not in use yet, initialize */ - if (state->wsize == 0) { - state->wsize = 1U << state->wbits; - state->wnext = 0; - state->whave = 0; - } + if (inflate_ensure_window(state)) return 1; /* copy state->wsize or less output bytes into the circular window */ if (copy >= state->wsize) { @@ -709,6 +727,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { if (flush == Z_BLOCK || flush == Z_TREES) goto inf_leave; /* fallthrough */ case TYPEDO: + INFLATE_TYPEDO_HOOK(strm, flush); if (state->last) { BYTEBITS(); state->mode = CHECK; @@ -1075,7 +1094,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { out -= left; strm->total_out += out; state->total += out; - if ((state->wrap & 4) && out) + if (INFLATE_NEED_CHECKSUM(strm) && (state->wrap & 4) && out) strm->adler = state->check = UPDATE_CHECK(state->check, put - out, out); out = left; @@ -1130,8 +1149,9 @@ int ZEXPORT inflate(z_streamp strm, int flush) { */ inf_leave: RESTORE(); - if (state->wsize || (out != strm->avail_out && state->mode < BAD && - (state->mode < CHECK || flush != Z_FINISH))) + if (INFLATE_NEED_UPDATEWINDOW(strm) && + (state->wsize || (out != strm->avail_out && state->mode < BAD && + (state->mode < CHECK || flush != Z_FINISH)))) if (updatewindow(strm, strm->next_out, out - strm->avail_out)) { state->mode = MEM; return Z_MEM_ERROR; @@ -1141,7 +1161,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { strm->total_in += in; strm->total_out += out; state->total += out; - if ((state->wrap & 4) && out) + if (INFLATE_NEED_CHECKSUM(strm) && (state->wrap & 4) && out) strm->adler = state->check = UPDATE_CHECK(state->check, strm->next_out - out, out); strm->data_type = (int)state->bits + (state->last ? 64 : 0) + @@ -1157,8 +1177,8 @@ int ZEXPORT inflateEnd(z_streamp strm) { if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; - if (state->window != Z_NULL) ZFREE(strm, state->window); - ZFREE(strm, strm->state); + if (state->window != Z_NULL) ZFREE_WINDOW(strm, state->window); + ZFREE_STATE(strm, strm->state); strm->state = Z_NULL; Tracev((stderr, "inflate: end\n")); return Z_OK; @@ -1172,6 +1192,8 @@ int ZEXPORT inflateGetDictionary(z_streamp strm, Bytef *dictionary, if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; + INFLATE_GET_DICTIONARY_HOOK(strm, dictionary, dictLength); + /* copy dictionary */ if (state->whave && dictionary != Z_NULL) { zmemcpy(dictionary, state->window + state->wnext, @@ -1204,6 +1226,8 @@ int ZEXPORT inflateSetDictionary(z_streamp strm, const Bytef *dictionary, return Z_DATA_ERROR; } + INFLATE_SET_DICTIONARY_HOOK(strm, dictionary, dictLength); + /* copy dictionary to window using updatewindow(), which will amend the existing dictionary if appropriate */ ret = updatewindow(strm, dictionary + dictLength, dictLength); @@ -1321,6 +1345,7 @@ int ZEXPORT inflateSyncPoint(z_streamp strm) { struct inflate_state FAR *state; if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + INFLATE_SYNC_POINT_HOOK(strm); state = (struct inflate_state FAR *)strm->state; return state->mode == STORED && state->bits == 0; } @@ -1337,22 +1362,23 @@ int ZEXPORT inflateCopy(z_streamp dest, z_streamp source) { /* allocate space */ copy = (struct inflate_state FAR *) - ZALLOC(source, 1, sizeof(struct inflate_state)); + ZALLOC_STATE(source, 1, sizeof(struct inflate_state)); if (copy == Z_NULL) return Z_MEM_ERROR; zmemzero(copy, sizeof(struct inflate_state)); window = Z_NULL; if (state->window != Z_NULL) { window = (unsigned char FAR *) - ZALLOC(source, 1U << state->wbits, sizeof(unsigned char)); + ZALLOC_WINDOW(source, 1U << state->wbits, + sizeof(unsigned char)); if (window == Z_NULL) { - ZFREE(source, copy); + ZFREE_STATE(source, copy); return Z_MEM_ERROR; } } /* copy state */ zmemcpy(dest, source, sizeof(z_stream)); - zmemcpy(copy, state, sizeof(struct inflate_state)); + ZCOPY_STATE(copy, state, sizeof(struct inflate_state)); copy->strm = dest; if (state->lencode >= state->codes && state->lencode <= state->codes + ENOUGH - 1) { @@ -1361,7 +1387,7 @@ int ZEXPORT inflateCopy(z_streamp dest, z_streamp source) { } copy->next = copy->codes + (state->next - state->codes); if (window != Z_NULL) - zmemcpy(window, state->window, state->whave); + ZCOPY_WINDOW(window, state->window, state->whave); copy->window = window; dest->state = (struct internal_state FAR *)copy; return Z_OK; @@ -1399,6 +1425,7 @@ long ZEXPORT inflateMark(z_streamp strm) { if (inflateStateCheck(strm)) return -(1L << 16); + INFLATE_MARK_HOOK(strm); state = (struct inflate_state FAR *)strm->state; return (long)(((unsigned long)((long)state->back)) << 16) + (state->mode == COPY ? state->length : diff --git a/inflate.h b/inflate.h index f758e0dcc1..62b6e5442a 100644 --- a/inflate.h +++ b/inflate.h @@ -124,3 +124,7 @@ struct inflate_state { int back; /* bits back of last unprocessed length/lit */ unsigned was; /* initial length of match */ }; + +#ifdef HAVE_S390X_DFLTCC +int ZLIB_INTERNAL inflate_ensure_window(struct inflate_state *state); +#endif diff --git a/trees.c b/trees.c index 8e4da01e9f..661036586b 100644 --- a/trees.c +++ b/trees.c @@ -151,7 +151,11 @@ local TCONST static_tree_desc static_bl_desc = * method would use a table) * IN assertion: 1 <= len <= 15 */ +#ifdef HAVE_S390X_DFLTCC +unsigned ZLIB_INTERNAL bi_reverse(unsigned code, int len) { +#else local unsigned bi_reverse(unsigned code, int len) { +#endif unsigned res = 0; do { res |= code & 1; @@ -178,7 +182,11 @@ local void bi_flush(deflate_state *s) { /* =========================================================================== * Flush the bit buffer and align the output on a byte boundary */ +#ifdef HAVE_S390X_DFLTCC +void ZLIB_INTERNAL bi_windup(deflate_state *s) { +#else local void bi_windup(deflate_state *s) { +#endif if (s->bi_valid > 8) { put_short(s, s->bi_buf); } else if (s->bi_valid > 0) { @@ -286,6 +294,9 @@ local void send_bits(deflate_state *s, int value, int length) { } #endif /* ZLIB_DEBUG */ +void ZLIB_INTERNAL _tr_send_bits(deflate_state *s, int value, int length) { + send_bits(s, value, length); +} /* the arguments must not have side effects */ diff --git a/zutil.h b/zutil.h index a9bc23ca62..6e3d81349a 100644 --- a/zutil.h +++ b/zutil.h @@ -13,7 +13,7 @@ #ifndef ZUTIL_H #define ZUTIL_H -#ifdef HAVE_HIDDEN +#if defined(HAVE_HIDDEN) && !defined (ZLIB_INTERNAL) # define ZLIB_INTERNAL __attribute__((visibility ("hidden"))) #else # define ZLIB_INTERNAL