diff -Nru libdeflate-1.7/common/compiler_gcc.h libdeflate-1.8/common/compiler_gcc.h --- libdeflate-1.7/common/compiler_gcc.h 2020-11-10 03:29:50.000000000 +0000 +++ libdeflate-1.8/common/compiler_gcc.h 2021-07-15 14:31:09.000000000 +0000 @@ -147,7 +147,8 @@ * needed. */ #if (GCC_PREREQ(4, 0) && !GCC_PREREQ(5, 1)) || \ - (defined(__clang__) && !CLANG_PREREQ(3, 9, 8020000)) + (defined(__clang__) && !CLANG_PREREQ(3, 9, 8020000)) || \ + defined(__INTEL_COMPILER) typedef unsigned long long __v2du __attribute__((__vector_size__(16))); typedef unsigned int __v4su __attribute__((__vector_size__(16))); typedef unsigned short __v8hu __attribute__((__vector_size__(16))); @@ -158,6 +159,12 @@ typedef unsigned char __v32qu __attribute__((__vector_size__(32))); #endif +#ifdef __INTEL_COMPILER +typedef int __v16si __attribute__((__vector_size__(64))); +typedef short __v32hi __attribute__((__vector_size__(64))); +typedef char __v64qi __attribute__((__vector_size__(64))); +#endif + /* Newer gcc supports __BYTE_ORDER__. Older gcc doesn't. */ #ifdef __BYTE_ORDER__ # define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) @@ -175,7 +182,16 @@ # define bswap64 __builtin_bswap64 #endif -#if defined(__x86_64__) || defined(__i386__) || defined(__ARM_FEATURE_UNALIGNED) || defined(__powerpc64__) +#if defined(__x86_64__) || defined(__i386__) || \ + defined(__ARM_FEATURE_UNALIGNED) || defined(__powerpc64__) || \ + /* + * For all compilation purposes, WebAssembly behaves like any other CPU + * instruction set. Even though WebAssembly engine might be running on top + * of different actual CPU architectures, the WebAssembly spec itself + * permits unaligned access and it will be fast on most of those platforms, + * and simulated at the engine level on others, so it's worth treating it + * as a CPU architecture with fast unaligned access. + */ defined(__wasm__) # define UNALIGNED_ACCESS_IS_FAST 1 #endif diff -Nru libdeflate-1.7/debian/changelog libdeflate-1.8/debian/changelog --- libdeflate-1.7/debian/changelog 2021-08-26 09:50:07.000000000 +0000 +++ libdeflate-1.8/debian/changelog 2021-11-14 23:40:38.000000000 +0000 @@ -1,3 +1,19 @@ +libdeflate (1.8-1ubuntu1) jammy; urgency=medium + + * Merge with Debian; remaining changes: + * lib/arm/crc32_impl.h: Use "arch=armv8-a+simd+crc" as the target arch. + + -- Robert Ancell Mon, 15 Nov 2021 12:40:38 +1300 + +libdeflate (1.8-1) unstable; urgency=medium + + * Team upload. + * Fix watchfile to detect new versions on github + * New upstream version + * Standards-Version: 4.6.0 (routine-update) + + -- Andreas Tille Sun, 10 Oct 2021 10:20:38 +0200 + libdeflate (1.7-2ubuntu2) impish; urgency=medium * Merge with Debian; remaining changes: diff -Nru libdeflate-1.7/debian/control libdeflate-1.8/debian/control --- libdeflate-1.7/debian/control 2021-08-26 09:50:07.000000000 +0000 +++ libdeflate-1.8/debian/control 2021-11-14 23:40:38.000000000 +0000 @@ -9,7 +9,7 @@ zlib1g-dev, python3 , xxd -Standards-Version: 4.5.1 +Standards-Version: 4.6.0 Vcs-Browser: https://salsa.debian.org/med-team/libdeflate Vcs-Git: https://salsa.debian.org/med-team/libdeflate.git Homepage: https://github.com/ebiggers/libdeflate diff -Nru libdeflate-1.7/debian/watch libdeflate-1.8/debian/watch --- libdeflate-1.7/debian/watch 2021-08-21 19:43:46.000000000 +0000 +++ libdeflate-1.8/debian/watch 2021-11-14 23:40:38.000000000 +0000 @@ -1,4 +1,4 @@ version=4 -https://github.com/ebiggers/libdeflate/releases/latest .*/archive/v?@ANY_VERSION@@ARCHIVE_EXT@ +https://github.com/ebiggers/libdeflate/releases/latest .*/v?@ANY_VERSION@@ARCHIVE_EXT@ # skips pre-releases diff -Nru libdeflate-1.7/.github/workflows/ci.yml libdeflate-1.8/.github/workflows/ci.yml --- libdeflate-1.7/.github/workflows/ci.yml 1970-01-01 00:00:00.000000000 +0000 +++ libdeflate-1.8/.github/workflows/ci.yml 2021-07-15 14:31:09.000000000 +0000 @@ -0,0 +1,123 @@ +name: CI +on: [pull_request] +env: + CFLAGS: -Werror + +jobs: + x86_64-build-and-test: + name: Build and test (x86_64, ${{ matrix.os }}, ${{ matrix.compiler }}) + strategy: + matrix: + os: [ubuntu-20.04, ubuntu-18.04, ubuntu-16.04] + compiler: [gcc, clang] + exclude: + # clang 3.8.0-2ubuntu4 crashes with: + # "fatal error: error in backend: Cannot select: 0x21025a0: v64i8 = X86ISD::VBROADCAST 0x2101fb0" + - os: ubuntu-16.04 + compiler: clang + runs-on: ${{ matrix.os }} + env: + CC: ${{ matrix.compiler }} + steps: + - uses: actions/checkout@v2 + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y clang llvm libz-dev valgrind + - run: scripts/run_tests.sh + + other-arch-build-and-test: + name: Build and test (${{ matrix.arch }}, Debian Buster, ${{ matrix.compiler }}) + strategy: + matrix: + arch: [armv6, armv7, aarch64, s390x, ppc64le] + compiler: [gcc, clang] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: uraimo/run-on-arch-action@v2.0.5 + with: + arch: ${{ matrix.arch }} + distro: buster + githubToken: ${{ github.token }} + install: | + apt-get update + apt-get install -y build-essential clang llvm libz-dev valgrind + run: | + # Valgrind and ASAN crash on at least s390x, ppc64le, and aarch64 + # here. (It's probably something related to the QEMU user-mode + # emulation that run-on-arch-action uses.) + export SKIP_VALGRIND=1 + export SKIP_ASAN=1 + + case ${{ matrix.arch }} in + s390x) + # On s390x, in freestanding builds the shared library links to an + # external symbol __clzdi2, even when -static-libgcc is used. + export SKIP_FREESTANDING=1 + ;; + aarch64) + # "ldd: exited with unknown exit code (139)" + if [ ${{ matrix.compiler }} = clang ]; then + export SKIP_SHARED_LIB=1 + fi + ;; + esac + + export CC=${{ matrix.compiler }} + scripts/run_tests.sh + + macos-build-and-test: + name: Build and test (macOS) + runs-on: macos-latest + steps: + - uses: actions/checkout@v2 + - run: make all check + + windows-build-and-test: + name: Build and test (Windows) + runs-on: windows-latest + steps: + - uses: actions/checkout@v2 + - shell: bash + run: | + PATH="C:\\msys64\\mingw64\\bin:C:\\msys64\\usr\\bin:$PATH" \ + make CC=gcc all check + + run-clang-static-analyzer: + name: Run clang static analyzer + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y clang-tools + - name: Run clang static analyzer + run: make scan-build + + run-shellcheck: + name: Run shellcheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y shellcheck + - name: Run shellcheck + run: make shellcheck + + cross-compile-for-windows: + name: Cross compile for Windows + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 libz-mingw-w64-dev + - name: 32-bit build + run: make CC=i686-w64-mingw32-gcc all test_programs + - name: 64-bit build + run: make CC=x86_64-w64-mingw32-gcc all test_programs diff -Nru libdeflate-1.7/lib/arm/adler32_impl.h libdeflate-1.8/lib/arm/adler32_impl.h --- libdeflate-1.7/lib/arm/adler32_impl.h 2020-11-10 03:29:50.000000000 +0000 +++ libdeflate-1.8/lib/arm/adler32_impl.h 2021-07-15 14:31:09.000000000 +0000 @@ -25,6 +25,9 @@ * OTHER DEALINGS IN THE SOFTWARE. */ +#ifndef LIB_ARM_ADLER32_IMPL_H +#define LIB_ARM_ADLER32_IMPL_H + #include "cpu_features.h" /* NEON implementation */ @@ -118,3 +121,5 @@ return NULL; } #endif /* DISPATCH */ + +#endif /* LIB_ARM_ADLER32_IMPL_H */ diff -Nru libdeflate-1.7/lib/arm/crc32_impl.h libdeflate-1.8/lib/arm/crc32_impl.h --- libdeflate-1.7/lib/arm/crc32_impl.h 2020-11-10 03:29:50.000000000 +0000 +++ libdeflate-1.8/lib/arm/crc32_impl.h 2021-07-15 14:31:09.000000000 +0000 @@ -26,6 +26,9 @@ * OTHER DEALINGS IN THE SOFTWARE. */ +#ifndef LIB_ARM_CRC32_IMPL_H +#define LIB_ARM_CRC32_IMPL_H + #include "cpu_features.h" /* Implementation using ARM CRC32 instructions */ @@ -240,3 +243,5 @@ return NULL; } #endif /* DISPATCH */ + +#endif /* LIB_ARM_CRC32_IMPL_H */ diff -Nru libdeflate-1.7/lib/arm/matchfinder_impl.h libdeflate-1.8/lib/arm/matchfinder_impl.h --- libdeflate-1.7/lib/arm/matchfinder_impl.h 2020-11-10 03:29:50.000000000 +0000 +++ libdeflate-1.8/lib/arm/matchfinder_impl.h 2021-07-15 14:31:09.000000000 +0000 @@ -25,6 +25,9 @@ * OTHER DEALINGS IN THE SOFTWARE. */ +#ifndef LIB_ARM_MATCHFINDER_IMPL_H +#define LIB_ARM_MATCHFINDER_IMPL_H + #ifdef __ARM_NEON # include static forceinline void @@ -79,3 +82,5 @@ #define matchfinder_rebase matchfinder_rebase_neon #endif /* __ARM_NEON */ + +#endif /* LIB_ARM_MATCHFINDER_IMPL_H */ diff -Nru libdeflate-1.7/lib/bt_matchfinder.h libdeflate-1.8/lib/bt_matchfinder.h --- libdeflate-1.7/lib/bt_matchfinder.h 2020-11-10 03:29:50.000000000 +0000 +++ libdeflate-1.8/lib/bt_matchfinder.h 2021-07-15 14:31:09.000000000 +0000 @@ -64,6 +64,8 @@ * ---------------------------------------------------------------------------- */ +#ifndef LIB_BT_MATCHFINDER_H +#define LIB_BT_MATCHFINDER_H #include "matchfinder_common.h" @@ -357,3 +359,5 @@ NULL, false); } + +#endif /* LIB_BT_MATCHFINDER_H */ diff -Nru libdeflate-1.7/lib/cpu_features_common.h libdeflate-1.8/lib/cpu_features_common.h --- libdeflate-1.7/lib/cpu_features_common.h 2020-11-10 03:29:50.000000000 +0000 +++ libdeflate-1.8/lib/cpu_features_common.h 2021-07-15 14:31:09.000000000 +0000 @@ -25,6 +25,9 @@ * OTHER DEALINGS IN THE SOFTWARE. */ +#ifndef LIB_CPU_FEATURES_COMMON_H +#define LIB_CPU_FEATURES_COMMON_H + #if defined(TEST_SUPPORT__DO_NOT_USE) && !defined(FREESTANDING) # define _GNU_SOURCE 1 /* for strdup() and strtok_r() */ # include @@ -81,3 +84,5 @@ { } #endif /* !TEST_SUPPORT__DO_NOT_USE */ + +#endif /* LIB_CPU_FEATURES_COMMON_H */ diff -Nru libdeflate-1.7/lib/hc_matchfinder.h libdeflate-1.8/lib/hc_matchfinder.h --- libdeflate-1.7/lib/hc_matchfinder.h 2020-11-10 03:29:50.000000000 +0000 +++ libdeflate-1.8/lib/hc_matchfinder.h 2021-07-15 14:31:09.000000000 +0000 @@ -106,6 +106,9 @@ * ---------------------------------------------------------------------------- */ +#ifndef LIB_HC_MATCHFINDER_H +#define LIB_HC_MATCHFINDER_H + #include "matchfinder_common.h" #define HC_MATCHFINDER_HASH3_ORDER 15 @@ -405,3 +408,5 @@ return in_next; } + +#endif /* LIB_HC_MATCHFINDER_H */ diff -Nru libdeflate-1.7/lib/utils.c libdeflate-1.8/lib/utils.c --- libdeflate-1.7/lib/utils.c 2020-11-10 03:29:50.000000000 +0000 +++ libdeflate-1.8/lib/utils.c 2021-07-15 14:31:09.000000000 +0000 @@ -85,7 +85,8 @@ */ #ifdef FREESTANDING #undef memset -void *memset(void *s, int c, size_t n) +void * __attribute__((weak)) +memset(void *s, int c, size_t n) { u8 *p = s; size_t i; @@ -96,7 +97,8 @@ } #undef memcpy -void *memcpy(void *dest, const void *src, size_t n) +void * __attribute__((weak)) +memcpy(void *dest, const void *src, size_t n) { u8 *d = dest; const u8 *s = src; @@ -108,7 +110,8 @@ } #undef memmove -void *memmove(void *dest, const void *src, size_t n) +void * __attribute__((weak)) +memmove(void *dest, const void *src, size_t n) { u8 *d = dest; const u8 *s = src; @@ -123,7 +126,8 @@ } #undef memcmp -int memcmp(const void *s1, const void *s2, size_t n) +int __attribute__((weak)) +memcmp(const void *s1, const void *s2, size_t n) { const u8 *p1 = s1; const u8 *p2 = s2; diff -Nru libdeflate-1.7/lib/x86/adler32_impl.h libdeflate-1.8/lib/x86/adler32_impl.h --- libdeflate-1.7/lib/x86/adler32_impl.h 2020-11-10 03:29:50.000000000 +0000 +++ libdeflate-1.8/lib/x86/adler32_impl.h 2021-07-15 14:31:09.000000000 +0000 @@ -25,6 +25,9 @@ * OTHER DEALINGS IN THE SOFTWARE. */ +#ifndef LIB_X86_ADLER32_IMPL_H +#define LIB_X86_ADLER32_IMPL_H + #include "cpu_features.h" /* @@ -330,3 +333,5 @@ return NULL; } #endif /* DISPATCH */ + +#endif /* LIB_X86_ADLER32_IMPL_H */ diff -Nru libdeflate-1.7/lib/x86/crc32_impl.h libdeflate-1.8/lib/x86/crc32_impl.h --- libdeflate-1.7/lib/x86/crc32_impl.h 2020-11-10 03:29:50.000000000 +0000 +++ libdeflate-1.8/lib/x86/crc32_impl.h 2021-07-15 14:31:09.000000000 +0000 @@ -25,6 +25,9 @@ * OTHER DEALINGS IN THE SOFTWARE. */ +#ifndef LIB_X86_CRC32_IMPL_H +#define LIB_X86_CRC32_IMPL_H + #include "cpu_features.h" /* @@ -85,3 +88,5 @@ return NULL; } #endif /* DISPATCH */ + +#endif /* LIB_X86_CRC32_IMPL_H */ diff -Nru libdeflate-1.7/lib/x86/crc32_pclmul_template.h libdeflate-1.8/lib/x86/crc32_pclmul_template.h --- libdeflate-1.7/lib/x86/crc32_pclmul_template.h 2020-11-10 03:29:50.000000000 +0000 +++ libdeflate-1.8/lib/x86/crc32_pclmul_template.h 2021-07-15 14:31:09.000000000 +0000 @@ -25,7 +25,7 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include +#include /* * CRC-32 folding with PCLMULQDQ. diff -Nru libdeflate-1.7/lib/x86/decompress_impl.h libdeflate-1.8/lib/x86/decompress_impl.h --- libdeflate-1.7/lib/x86/decompress_impl.h 2020-11-10 03:29:50.000000000 +0000 +++ libdeflate-1.8/lib/x86/decompress_impl.h 2021-07-15 14:31:09.000000000 +0000 @@ -1,3 +1,6 @@ +#ifndef LIB_X86_DECOMPRESS_IMPL_H +#define LIB_X86_DECOMPRESS_IMPL_H + #include "cpu_features.h" /* Include the BMI2-optimized version? */ @@ -24,3 +27,5 @@ return NULL; } #endif /* DISPATCH */ + +#endif /* LIB_X86_DECOMPRESS_IMPL_H */ diff -Nru libdeflate-1.7/lib/x86/matchfinder_impl.h libdeflate-1.8/lib/x86/matchfinder_impl.h --- libdeflate-1.7/lib/x86/matchfinder_impl.h 2020-11-10 03:29:50.000000000 +0000 +++ libdeflate-1.8/lib/x86/matchfinder_impl.h 2021-07-15 14:31:09.000000000 +0000 @@ -25,6 +25,9 @@ * OTHER DEALINGS IN THE SOFTWARE. */ +#ifndef LIB_X86_MATCHFINDER_IMPL_H +#define LIB_X86_MATCHFINDER_IMPL_H + #ifdef __AVX2__ # include static forceinline void @@ -115,3 +118,5 @@ } #define matchfinder_rebase matchfinder_rebase_sse2 #endif /* __SSE2__ */ + +#endif /* LIB_X86_MATCHFINDER_IMPL_H */ diff -Nru libdeflate-1.7/libdeflate.h libdeflate-1.8/libdeflate.h --- libdeflate-1.7/libdeflate.h 2020-11-10 03:29:50.000000000 +0000 +++ libdeflate-1.8/libdeflate.h 2021-07-15 14:31:09.000000000 +0000 @@ -10,8 +10,8 @@ #endif #define LIBDEFLATE_VERSION_MAJOR 1 -#define LIBDEFLATE_VERSION_MINOR 7 -#define LIBDEFLATE_VERSION_STRING "1.7" +#define LIBDEFLATE_VERSION_MINOR 8 +#define LIBDEFLATE_VERSION_STRING "1.8" #include #include diff -Nru libdeflate-1.7/NEWS libdeflate-1.8/NEWS --- libdeflate-1.7/NEWS 2020-11-10 03:29:50.000000000 +0000 +++ libdeflate-1.8/NEWS 1970-01-01 00:00:00.000000000 +0000 @@ -1,200 +0,0 @@ -Version 1.7: - Added support for compression level 0, "no compression". - - Added an ARM CRC32 instruction accelerated implementation of CRC32. - - Added support for linking the programs to the shared library version of - libdeflate rather than to the static library version. - - Made the compression level affect the minimum input size at which - compression is attempted. - - Fixed undefined behavior in x86 Adler32 implementation. - (No miscompilations were observed in practice.) - - Fixed undefined behavior in x86 CPU feature code. - (No miscompilations were observed in practice.) - - Fixed installing shared lib symlink on macOS. - - Documented third-party bindings. - - Made a lot of improvements to the testing scripts and the CI - configuration file. - - Lots of other small improvements and cleanups. - -Version 1.6: - Prevented gcc 10 from miscompiling libdeflate (workaround for - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=94994). - - Removed workaround for gcc 5 and earlier producing slow code on - ARM32. If this affects you, please upgrade your compiler. - - New API function: libdeflate_zlib_decompress_ex(). It provides - the actual size of the stream that was decompressed, like the - gzip and DEFLATE equivalents. - - libdeflate_zlib_decompress() now accepts trailing bytes after - the end of the stream, like the gzip and DEFLATE equivalents. - - Added support for custom memory allocators. - (New API function: libdeflate_set_memory_allocator()) - - Added support for building the library in freestanding mode. - - Building libdeflate no longer requires CPPFLAGS=-Icommon. - -Version 1.5: - Fixed up stdcall support on 32-bit Windows: the functions are - now exported using both suffixed and non-suffixed names, and - fixed libdeflate.h to be MSVC-compatible again. - -Version 1.4: - The 32-bit Windows build of libdeflate now uses the "stdcall" - calling convention instead of "cdecl". If you're calling - libdeflate.dll directly from C or C++, you'll need to recompile - your code. If you're calling it from another language, or - calling it indirectly using LoadLibrary(), you'll need to update - your code to use the stdcall calling convention. - - The Makefile now supports building libdeflate as a shared - library (.dylib) on macOS. - - Fixed a bug where support for certain optimizations and optional - features (file access hints and more precise timestamps) was - incorrectly omitted when libdeflate was compiled with -Werror. - - Added 'make check' target to the Makefile. - - Added CI configuration files. - -Version 1.3: - `make install` now supports customizing the directories into - which binaries, headers, and libraries are installed. - - `make install` now installs into /usr/local by default. - To change it, use e.g. `make install PREFIX=/usr`. - - `make install` now works on more platforms. - - The Makefile now supports overriding the optimization flags. - - The compression functions now correctly handle an output data - buffer >= 4 GiB in size, and `gzip` and `gunzip` now correctly - handle multi-gigabyte files (if enough memory is available). - -Version 1.2: - Slight improvements to decompression speed. - - Added an AVX-512BW implementation of Adler-32. - - The Makefile now supports a user-specified installation PREFIX. - - Fixed build error with some Visual Studio versions. - -Version 1.1: - Fixed crash in CRC-32 code when the prebuilt libdeflate for - 32-bit Windows was called by a program built with Visual Studio. - - Improved the worst-case decompression speed of malicious data. - - Fixed build error when compiling for an ARM processor without - hardware floating point support. - - Improved performance on the PowerPC64 architecture. - - Added soname to libdeflate.so, to make packaging easier. - - Added 'make install' target to the Makefile. - - The Makefile now supports user-specified CPPFLAGS. - - The Windows binary releases now include the import library for - libdeflate.dll. libdeflate.lib is now the import library, and - libdeflatestatic.lib is the static library. - -Version 1.0: - Added support for multi-member gzip files. - - Moved architecture-specific code into subdirectories. If you - aren't using the provided Makefile to build libdeflate, you now - need to compile lib/*.c and lib/*/*.c instead of just lib/*.c. - - Added an ARM PMULL implementation of CRC-32, which speeds up - gzip compression and decompression on 32-bit and 64-bit ARM - processors that have the Cryptography Extensions. - - Improved detection of CPU features, resulting in accelerated - functions being used in more cases. This includes: - - - Detect CPU features on 32-bit x86, not just 64-bit as was - done previously. - - - Detect CPU features on ARM, both 32 and 64-bit. - (Limited to Linux only currently.) - -Version 0.8: - Build fixes for certain platforms and compilers. - - libdeflate now produces the same output on all CPU architectures. - - Improved documentation for building libdeflate on Windows. - -Version 0.7: - Fixed a very rare bug that caused data to be compressed incorrectly. - The bug affected compression levels 7 and below since libdeflate v0.2. - Although there have been no user reports of the bug, and I believe it - would have been highly unlikely to encounter on realistic data, it could - occur on data specially crafted to reproduce it. - - Fixed a compilation error when building with clang 3.7. - -Version 0.6: - Various improvements to the gzip program's behavior. - - Faster CRC-32 on AVX-capable processors. - - Other minor changes. - -Version 0.5: - The CRC-32 checksum algorithm has been optimized with carryless - multiplication instructions for x86_64 (PCLMUL). This speeds up gzip - compression and decompression. - - Build fixes for certain platforms and compilers. - - Added more test programs and scripts. - - libdeflate is now entirely MIT-licensed. - -Version 0.4: - The Adler-32 checksum algorithm has been optimized with vector - instructions for x86_64 (SSE2 and AVX2) and ARM (NEON). This speeds up - zlib compression and decompression. - - To avoid naming collisions, functions and definitions in libdeflate's - API have been renamed to be prefixed with "libdeflate_" or - "LIBDEFLATE_". Programs using the old API will need to be updated. - - Various bug fixes and other improvements. - -Version 0.3: - Some bug fixes and other minor changes. - -Version 0.2: - Implemented a new block splitting algorithm which typically improves the - compression ratio slightly at all compression levels. - - The compressor now outputs each block using the cheapest type (dynamic - Huffman, static Huffman, or uncompressed). - - The gzip program has received an overhaul and now behaves more like the - standard version. - - Build system updates, including: some build options were changed and - some build options were removed, and the default 'make' target now - includes the gzip program as well as the library. - -Version 0.1: - Initial official release. diff -Nru libdeflate-1.7/NEWS.md libdeflate-1.8/NEWS.md --- libdeflate-1.7/NEWS.md 1970-01-01 00:00:00.000000000 +0000 +++ libdeflate-1.8/NEWS.md 2021-07-15 14:31:09.000000000 +0000 @@ -0,0 +1,230 @@ +# libdeflate release notes + +## Version 1.8 + +* Added `-t` (test) option to `libdeflate-gunzip`. + +* Unaligned access optimizations are now enabled on WebAssembly builds. + +* Fixed a build error when building with the Intel C Compiler (ICC). + +* Fixed a build error when building with uClibc. + +* libdeflate's CI system has switched from Travis CI to GitHub Actions. + +* Made some improvements to test scripts. + +## Version 1.7 + +* Added support for compression level 0, "no compression". + +* Added an ARM CRC32 instruction accelerated implementation of CRC32. + +* Added support for linking the programs to the shared library version of + libdeflate rather than to the static library version. + +* Made the compression level affect the minimum input size at which compression + is attempted. + +* Fixed undefined behavior in x86 Adler32 implementation. (No miscompilations + were observed in practice.) + +* Fixed undefined behavior in x86 CPU feature code. (No miscompilations were + observed in practice.) + +* Fixed installing shared lib symlink on macOS. + +* Documented third-party bindings. + +* Made a lot of improvements to the testing scripts and the CI configuration + file. + +* Lots of other small improvements and cleanups. + +## Version 1.6 + +* Prevented gcc 10 from miscompiling libdeflate (workaround for + https://gcc.gnu.org/bugzilla/show_bug.cgi?id=94994). + +* Removed workaround for gcc 5 and earlier producing slow code on ARM32. If + this affects you, please upgrade your compiler. + +* New API function: `libdeflate_zlib_decompress_ex()`. It provides the actual + size of the stream that was decompressed, like the gzip and DEFLATE + equivalents. + +* `libdeflate_zlib_decompress()` now accepts trailing bytes after the end of the + stream, like the gzip and DEFLATE equivalents. + +* Added support for custom memory allocators. (New API function: + `libdeflate_set_memory_allocator()`) + +* Added support for building the library in freestanding mode. + +* Building libdeflate no longer requires `CPPFLAGS=-Icommon`. + +## Version 1.5 + +* Fixed up stdcall support on 32-bit Windows: the functions are now exported + using both suffixed and non-suffixed names, and fixed `libdeflate.h` to be + MSVC-compatible again. + +## Version 1.4 + +* The 32-bit Windows build of libdeflate now uses the "stdcall" calling + convention instead of "cdecl". If you're calling `libdeflate.dll` directly + from C or C++, you'll need to recompile your code. If you're calling it from + another language, or calling it indirectly using `LoadLibrary()`, you'll need + to update your code to use the stdcall calling convention. + +* The Makefile now supports building libdeflate as a shared + library (`.dylib`) on macOS. + +* Fixed a bug where support for certain optimizations and optional features + (file access hints and more precise timestamps) was incorrectly omitted when + libdeflate was compiled with `-Werror`. + +* Added `make check` target to the Makefile. + +* Added CI configuration files. + +## Version 1.3 + +* `make install` now supports customizing the directories into which binaries, + headers, and libraries are installed. + +* `make install` now installs into `/usr/local` by default. To change it, use + e.g. `make install PREFIX=/usr`. + +* `make install` now works on more platforms. + +* The Makefile now supports overriding the optimization flags. + +* The compression functions now correctly handle an output data buffer >= 4 GiB + in size, and `gzip` and `gunzip` now correctly handle multi-gigabyte files (if + enough memory is available). + +## Version 1.2 + +* Slight improvements to decompression speed. + +* Added an AVX-512BW implementation of Adler-32. + +* The Makefile now supports a user-specified installation `PREFIX`. + +* Fixed build error with some Visual Studio versions. + +## Version 1.1 + +* Fixed crash in CRC-32 code when the prebuilt libdeflate for 32-bit Windows was + called by a program built with Visual Studio. + +* Improved the worst-case decompression speed of malicious data. + +* Fixed build error when compiling for an ARM processor without hardware + floating point support. + +* Improved performance on the PowerPC64 architecture. + +* Added soname to `libdeflate.so`, to make packaging easier. + +* Added `make install` target to the Makefile. + +* The Makefile now supports user-specified `CPPFLAGS`. + +* The Windows binary releases now include the import library for + `libdeflate.dll`. `libdeflate.lib` is now the import library, and + `libdeflatestatic.lib` is the static library. + +## Version 1.0 + +* Added support for multi-member gzip files. + +* Moved architecture-specific code into subdirectories. If you aren't using the + provided Makefile to build libdeflate, you now need to compile `lib/*.c` and + `lib/*/*.c` instead of just `lib/*.c`. + +* Added an ARM PMULL implementation of CRC-32, which speeds up gzip compression + and decompression on 32-bit and 64-bit ARM processors that have the + Cryptography Extensions. + +* Improved detection of CPU features, resulting in accelerated functions being + used in more cases. This includes: + + * Detect CPU features on 32-bit x86, not just 64-bit as was done previously. + + * Detect CPU features on ARM, both 32 and 64-bit. (Limited to Linux only + currently.) + +## Version 0.8 + +* Build fixes for certain platforms and compilers. + +* libdeflate now produces the same output on all CPU architectures. + +* Improved documentation for building libdeflate on Windows. + +## Version 0.7 + +* Fixed a very rare bug that caused data to be compressed incorrectly. The bug + affected compression levels 7 and below since libdeflate v0.2. Although there + have been no user reports of the bug, and I believe it would have been highly + unlikely to encounter on realistic data, it could occur on data specially + crafted to reproduce it. + +* Fixed a compilation error when building with clang 3.7. + +## Version 0.6 + +* Various improvements to the gzip program's behavior. + +* Faster CRC-32 on AVX-capable processors. + +* Other minor changes. + +## Version 0.5 + +* The CRC-32 checksum algorithm has been optimized with carryless multiplication + instructions for `x86_64` (PCLMUL). This speeds up gzip compression and + decompression. + +* Build fixes for certain platforms and compilers. + +* Added more test programs and scripts. + +* libdeflate is now entirely MIT-licensed. + +## Version 0.4 + +* The Adler-32 checksum algorithm has been optimized with vector instructions + for `x86_64` (SSE2 and AVX2) and ARM (NEON). This speeds up zlib compression + and decompression. + +* To avoid naming collisions, functions and definitions in libdeflate's API have + been renamed to be prefixed with `libdeflate_` or `LIBDEFLATE_`. Programs + using the old API will need to be updated. + +* Various bug fixes and other improvements. + +## Version 0.3 + +* Some bug fixes and other minor changes. + +## Version 0.2 + +* Implemented a new block splitting algorithm which typically improves the + compression ratio slightly at all compression levels. + +* The compressor now outputs each block using the cheapest type (dynamic + Huffman, static Huffman, or uncompressed). + +* The gzip program has received an overhaul and now behaves more like the + standard version. + +* Build system updates, including: some build options were changed and some + build options were removed, and the default 'make' target now includes the + gzip program as well as the library. + +## Version 0.1 + +* Initial official release. diff -Nru libdeflate-1.7/programs/benchmark.c libdeflate-1.8/programs/benchmark.c --- libdeflate-1.7/programs/benchmark.c 2020-11-10 03:29:50.000000000 +0000 +++ libdeflate-1.8/programs/benchmark.c 2021-07-15 14:31:09.000000000 +0000 @@ -29,21 +29,21 @@ static const tchar *const optstring = T("0::1::2::3::4::5::6::7::8::9::C:D:eghs:VYZz"); -enum wrapper { - NO_WRAPPER, - ZLIB_WRAPPER, - GZIP_WRAPPER, +enum format { + DEFLATE_FORMAT, + ZLIB_FORMAT, + GZIP_FORMAT, }; struct compressor { int level; - enum wrapper wrapper; + enum format format; const struct engine *engine; void *private; }; struct decompressor { - enum wrapper wrapper; + enum format format; const struct engine *engine; void *private; }; @@ -75,10 +75,10 @@ static size_t libdeflate_engine_compress_bound(struct compressor *c, size_t in_nbytes) { - switch (c->wrapper) { - case ZLIB_WRAPPER: + switch (c->format) { + case ZLIB_FORMAT: return libdeflate_zlib_compress_bound(c->private, in_nbytes); - case GZIP_WRAPPER: + case GZIP_FORMAT: return libdeflate_gzip_compress_bound(c->private, in_nbytes); default: return libdeflate_deflate_compress_bound(c->private, in_nbytes); @@ -89,11 +89,11 @@ libdeflate_engine_compress(struct compressor *c, const void *in, size_t in_nbytes, void *out, size_t out_nbytes_avail) { - switch (c->wrapper) { - case ZLIB_WRAPPER: + switch (c->format) { + case ZLIB_FORMAT: return libdeflate_zlib_compress(c->private, in, in_nbytes, out, out_nbytes_avail); - case GZIP_WRAPPER: + case GZIP_FORMAT: return libdeflate_gzip_compress(c->private, in, in_nbytes, out, out_nbytes_avail); default: @@ -119,11 +119,11 @@ libdeflate_engine_decompress(struct decompressor *d, const void *in, size_t in_nbytes, void *out, size_t out_nbytes) { - switch (d->wrapper) { - case ZLIB_WRAPPER: + switch (d->format) { + case ZLIB_FORMAT: return !libdeflate_zlib_decompress(d->private, in, in_nbytes, out, out_nbytes, NULL); - case GZIP_WRAPPER: + case GZIP_FORMAT: return !libdeflate_gzip_decompress(d->private, in, in_nbytes, out, out_nbytes, NULL); default: @@ -154,13 +154,13 @@ /******************************************************************************/ static int -get_libz_window_bits(enum wrapper wrapper) +get_libz_window_bits(enum format format) { const int windowBits = 15; - switch (wrapper) { - case ZLIB_WRAPPER: + switch (format) { + case ZLIB_FORMAT: return windowBits; - case GZIP_WRAPPER: + case GZIP_FORMAT: return windowBits + 16; default: return -windowBits; @@ -187,7 +187,7 @@ z->zfree = NULL; z->opaque = NULL; if (deflateInit2(z, c->level, Z_DEFLATED, - get_libz_window_bits(c->wrapper), + get_libz_window_bits(c->format), 8, Z_DEFAULT_STRATEGY) != Z_OK) { msg("unable to initialize deflater"); @@ -247,7 +247,7 @@ z->zalloc = NULL; z->zfree = NULL; z->opaque = NULL; - if (inflateInit2(z, get_libz_window_bits(d->wrapper)) != Z_OK) { + if (inflateInit2(z, get_libz_window_bits(d->format)) != Z_OK) { msg("unable to initialize inflater"); free(z); return false; @@ -318,11 +318,11 @@ /******************************************************************************/ static bool -compressor_init(struct compressor *c, int level, enum wrapper wrapper, +compressor_init(struct compressor *c, int level, enum format format, const struct engine *engine) { c->level = level; - c->wrapper = wrapper; + c->format = format; c->engine = engine; return engine->init_compressor(c); } @@ -348,10 +348,10 @@ } static bool -decompressor_init(struct decompressor *d, enum wrapper wrapper, +decompressor_init(struct decompressor *d, enum format format, const struct engine *engine) { - d->wrapper = wrapper; + d->format = format; d->engine = engine; return engine->init_decompressor(d); } @@ -401,11 +401,11 @@ " -C ENGINE compression engine\n" " -D ENGINE decompression engine\n" " -e allow chunks to be expanded (implied by -0)\n" -" -g use gzip wrapper\n" +" -g use gzip format instead of raw DEFLATE\n" " -h print this help\n" " -s SIZE chunk size\n" " -V show version and legal information\n" -" -z use zlib wrapper\n" +" -z use zlib format instead of raw DEFLATE\n" "\n", prog_invocation_name); show_available_engines(fp); @@ -542,7 +542,7 @@ { u32 chunk_size = 1048576; int level = 6; - enum wrapper wrapper = NO_WRAPPER; + enum format format = DEFLATE_FORMAT; const struct engine *compress_engine = &DEFAULT_ENGINE; const struct engine *decompress_engine = &DEFAULT_ENGINE; bool allow_expansion = false; @@ -595,7 +595,7 @@ allow_expansion = true; break; case 'g': - wrapper = GZIP_WRAPPER; + format = GZIP_FORMAT; break; case 'h': show_usage(stdout); @@ -617,7 +617,7 @@ decompress_engine = &libz_engine; break; case 'z': - wrapper = ZLIB_WRAPPER; + format = ZLIB_FORMAT; break; default: show_usage(stderr); @@ -632,9 +632,9 @@ allow_expansion = true; ret = -1; - if (!compressor_init(&compressor, level, wrapper, compress_engine)) + if (!compressor_init(&compressor, level, format, compress_engine)) goto out; - if (!decompressor_init(&decompressor, wrapper, decompress_engine)) + if (!decompressor_init(&decompressor, format, decompress_engine)) goto out; if (allow_expansion) @@ -660,12 +660,11 @@ argv[i] = NULL; } - printf("Benchmarking DEFLATE compression:\n"); + printf("Benchmarking %s compression:\n", + format == DEFLATE_FORMAT ? "DEFLATE" : + format == ZLIB_FORMAT ? "zlib" : "gzip"); printf("\tCompression level: %d\n", level); printf("\tChunk size: %"PRIu32"\n", chunk_size); - printf("\tWrapper: %s\n", - wrapper == NO_WRAPPER ? "None" : - wrapper == ZLIB_WRAPPER ? "zlib" : "gzip"); printf("\tCompression engine: %"TS"\n", compress_engine->name); printf("\tDecompression engine: %"TS"\n", decompress_engine->name); diff -Nru libdeflate-1.7/programs/gzip.c libdeflate-1.8/programs/gzip.c --- libdeflate-1.7/programs/gzip.c 2020-11-10 03:29:50.000000000 +0000 +++ libdeflate-1.8/programs/gzip.c 2021-07-15 14:31:09.000000000 +0000 @@ -43,11 +43,12 @@ bool decompress; bool force; bool keep; + bool test; int compression_level; const tchar *suffix; }; -static const tchar *const optstring = T("1::2::3::4::5::6::7::8::9::cdfhknS:V"); +static const tchar *const optstring = T("1::2::3::4::5::6::7::8::9::cdfhknS:tV"); static void show_usage(FILE *fp) @@ -66,6 +67,7 @@ " -h print this help\n" " -k don't delete input files\n" " -S SUF use suffix SUF instead of .gz\n" +" -t test file integrity\n" " -V show version and legal information\n", prog_invocation_name); } @@ -183,7 +185,8 @@ static int do_decompress(struct libdeflate_decompressor *decompressor, - struct file_stream *in, struct file_stream *out) + struct file_stream *in, struct file_stream *out, + const struct options *options) { const u8 *compressed_data = in->mmap_mem; size_t compressed_size = in->mmap_size; @@ -258,9 +261,11 @@ goto out; } - ret = full_write(out, uncompressed_data, actual_out_nbytes); - if (ret != 0) - goto out; + if (!options->test) { + ret = full_write(out, uncompressed_data, actual_out_nbytes); + if (ret != 0) + goto out; + } compressed_data += actual_in_nbytes; compressed_size -= actual_in_nbytes; @@ -425,7 +430,7 @@ if (ret != 0) goto out_close_out; - ret = do_decompress(decompressor, &in, &out); + ret = do_decompress(decompressor, &in, &out, options); if (ret != 0) goto out_close_out; @@ -534,6 +539,7 @@ options.decompress = is_gunzip(); options.force = false; options.keep = false; + options.test = false; options.compression_level = 6; options.suffix = T(".gz"); @@ -583,6 +589,17 @@ return 1; } break; + case 't': + options.test = true; + options.decompress = true; + options.to_stdout = true; + /* + * -t behaves just like the more commonly used -c + * option, except that -t doesn't actually write + * anything. For ease of implementation, just pretend + * that -c was specified too. + */ + break; case 'V': show_version(); return 0; diff -Nru libdeflate-1.7/programs/prog_util.c libdeflate-1.8/programs/prog_util.c --- libdeflate-1.7/programs/prog_util.c 2020-11-10 03:29:50.000000000 +0000 +++ libdeflate-1.8/programs/prog_util.c 2021-07-15 14:31:09.000000000 +0000 @@ -186,7 +186,7 @@ } #if defined(HAVE_POSIX_FADVISE) && (O_SEQUENTIAL == 0) - posix_fadvise(strm->fd, 0, 0, POSIX_FADV_SEQUENTIAL); + (void)posix_fadvise(strm->fd, 0, 0, POSIX_FADV_SEQUENTIAL); #endif return 0; @@ -347,7 +347,7 @@ } #ifdef HAVE_POSIX_MADVISE - posix_madvise(strm->mmap_mem, size, POSIX_MADV_SEQUENTIAL); + (void)posix_madvise(strm->mmap_mem, size, POSIX_MADV_SEQUENTIAL); #endif strm->mmap_token = strm; /* anything that's not NULL */ diff -Nru libdeflate-1.7/programs/prog_util.h libdeflate-1.8/programs/prog_util.h --- libdeflate-1.7/programs/prog_util.h 2020-11-10 03:29:50.000000000 +0000 +++ libdeflate-1.8/programs/prog_util.h 2021-07-15 14:31:09.000000000 +0000 @@ -38,6 +38,9 @@ #include #include #include +#ifndef _WIN32 +# include +#endif #include "../common/common_defs.h" diff -Nru libdeflate-1.7/README.md libdeflate-1.8/README.md --- libdeflate-1.7/README.md 2020-11-10 03:29:50.000000000 +0000 +++ libdeflate-1.8/README.md 2021-07-15 14:31:09.000000000 +0000 @@ -1,5 +1,3 @@ -[![Build Status](https://travis-ci.org/ebiggers/libdeflate.svg?branch=master)](https://travis-ci.org/ebiggers/libdeflate) - # Overview libdeflate is a library for fast, whole-buffer DEFLATE-based compression and @@ -24,6 +22,8 @@ yet support very large files * benchmark, a program for benchmarking in-memory compression and decompression +For the release notes, see the [NEWS file](NEWS.md). + ## Table of Contents - [Building](#building) @@ -194,6 +194,7 @@ libdeflate from a programming language other than C or C++, consider using the following bindings: +* C#: [LibDeflate.NET](https://github.com/jzebedee/LibDeflate.NET) * Go: [go-libdeflate](https://github.com/4kills/go-libdeflate) * Java: [libdeflate-java](https://github.com/astei/libdeflate-java) * Julia: [LibDeflate.jl](https://github.com/jakobnissen/LibDeflate.jl) diff -Nru libdeflate-1.7/scripts/android_tests.sh libdeflate-1.8/scripts/android_tests.sh --- libdeflate-1.7/scripts/android_tests.sh 2020-11-10 03:29:50.000000000 +0000 +++ libdeflate-1.8/scripts/android_tests.sh 2021-07-15 14:31:09.000000000 +0000 @@ -21,19 +21,26 @@ exit 1 fi +CLEANUP_CMDS=() +cleanup() { + for cmd in "${CLEANUP_CMDS[@]}"; do + eval "$cmd" + done +} +trap cleanup EXIT + # Use TESTDATA if specified in environment, else generate it. if [ -z "${TESTDATA:-}" ]; then # Generate default TESTDATA file. - TESTDATA=$(mktemp -t libdeflate_testdata.XXXXXXX) + TESTDATA=$(mktemp -t libdeflate_testdata.XXXXXXXXXX) export TESTDATA - trap 'rm -f "$TESTDATA"' EXIT + CLEANUP_CMDS+=("rm -f '$TESTDATA'") find . '(' -name '*.c' -o -name '*.h' -o -name '*.sh' ')' \ -exec cat '{}' ';' | head -c 1000000 > "$TESTDATA" fi -# Create a temporary file. -TMPFILE=$(mktemp -t libdeflate_tmpfile.XXXXXX) -trap 'rm -f "$TMPFILE"' EXIT +TMPDIR=$(mktemp -d -t libdeflate_test.XXXXXXXXX) +CLEANUP_CMDS+=("rm -r '$TMPDIR'") android_build_and_test() { echo "Running Android tests with $*" @@ -45,10 +52,10 @@ # Note: adb shell always returns 0, even if the shell command fails... adb shell "cd /data/local/tmp && WRAPPER= TESTDATA=$(basename "$TESTDATA") sh exec_tests.sh" \ - > "$TMPFILE" - if ! grep -q "exec_tests finished successfully" "$TMPFILE"; then + > "$TMPDIR/adb.out" + if ! grep -q "exec_tests finished successfully" "$TMPDIR/adb.out"; then echo 1>&2 "Android test failure! adb shell output:" - cat "$TMPFILE" + cat "$TMPDIR/adb.out" exit 1 fi } diff -Nru libdeflate-1.7/scripts/gzip_tests.sh libdeflate-1.8/scripts/gzip_tests.sh --- libdeflate-1.7/scripts/gzip_tests.sh 2020-11-10 03:29:50.000000000 +0000 +++ libdeflate-1.8/scripts/gzip_tests.sh 2021-07-15 14:31:09.000000000 +0000 @@ -363,19 +363,21 @@ cmp 2 file -begin_test 'Multiple files, continue on error' -cp file 1 -cp file 2 -chmod a-r 1 -assert_error 'Permission denied' gzip 1 2 -[ ! -e 1.gz ] -cmp file <(gunzip -c 2.gz) -rm -f 1 -cp 2.gz 1.gz -chmod a-r 1.gz -assert_error 'Permission denied' gunzip 1.gz 2.gz -[ ! -e 1 ] -cmp 2 file +if (( $(id -u) != 0 )); then + begin_test 'Multiple files, continue on error' + cp file 1 + cp file 2 + chmod a-r 1 + assert_error 'Permission denied' gzip 1 2 + [ ! -e 1.gz ] + cmp file <(gunzip -c 2.gz) + rm -f 1 + cp 2.gz 1.gz + chmod a-r 1.gz + assert_error 'Permission denied' gunzip 1.gz 2.gz + [ ! -e 1 ] + cmp 2 file +fi begin_test 'Compressing empty file' @@ -393,10 +395,8 @@ assert_error '\' gunzip foo.gz echo abcdefgh > foo.gz assert_error '\' gunzip foo.gz -xxd -r > foo.gz <<-EOF -00000000: 1f8b 0800 0000 0000 00ff 4b4c 4a4e 4924 ..........KLJNI$ -00000010: 1673 0100 6c5b a262 2e00 0000 .s..l[.b.... -EOF +echo -ne '\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x4b\x4c\x4a\x4e\x49\x24\x16\x73\x01\x00\x6c\x5b\xa2\x62\x2e\x00\x00\x00' \ + > foo.gz assert_error '\<(not in gzip format|crc error)\>' gunzip foo.gz @@ -463,6 +463,26 @@ done +begin_test '-t (test) option works' +good_files=( +'H4sIAAAAAAAAA3PMSVTITVTIzi9JVABTIJ5jzpGZelwAX+86ehsAAAA=' +'H4sIAAAAAAAAAwvJSFUoLM1MzlZIKsovz1NIy69QyCrNLShWyC9LLVIoAUrnJFZVKqTkp+txAQBqzFDrLQAAAA==') +bad_files=( +'H4sIAO1YYmAAA3PMSVTITVTIzi9JVABTIJ5jzpGZelwAX+46ehsAAAA=' +'H4sIAO1YYmAAA3PMSVTITVTIzi85VABTIJ5jzpGZelwAX+86ehsAAAA=' +'H4sIAAAAAAAAA3PMSVTITVTIzi9JVABTIJ5jzpGZelwAX+86ehsBAAA=' +'H4sIAAAAAAAAAwvJSFUoLM1MzlZIKsovz1NIy69QyCrNLShWyC9LLVIogUrnJFZVKqTkp+txAQBqzFDrLQAAAA==' +'H4sIAAAAAAAAAwvJSFUoLM1MzlZIKsovz1NIy69QyCrNLShWyC9L') +for contents in "${good_files[@]}"; do + echo "$contents" | base64 -d | gzip -t +done +for contents in "${bad_files[@]}"; do + echo "$contents" | base64 -d > file + assert_error '\' \ + gzip -t file +done + + begin_test 'Version information' gzip -V | grep -q Copyright gunzip -V | grep -q Copyright diff -Nru libdeflate-1.7/scripts/make-windows-releases.sh libdeflate-1.8/scripts/make-windows-releases.sh --- libdeflate-1.7/scripts/make-windows-releases.sh 2020-11-10 03:29:50.000000000 +0000 +++ libdeflate-1.8/scripts/make-windows-releases.sh 2021-07-15 14:31:09.000000000 +0000 @@ -12,9 +12,8 @@ cp libdeflate.{dll,lib,def} libdeflatestatic.lib libdeflate.h ./*.exe \ "$dir" ${arch}-w64-mingw32-strip "$dir/libdeflate.dll" "$dir"/*.exe - for file in COPYING NEWS; do + for file in COPYING NEWS.md README.md; do sed < $file > "$dir/${file}.txt" -e 's/$/\r/g' done - sed < README.md > "$dir/README.md" -e 's/$/\r/g' (cd "$dir" && zip -r "../${dir}.zip" .) done diff -Nru libdeflate-1.7/scripts/run_tests.sh libdeflate-1.8/scripts/run_tests.sh --- libdeflate-1.7/scripts/run_tests.sh 2020-11-10 03:29:50.000000000 +0000 +++ libdeflate-1.8/scripts/run_tests.sh 2021-07-15 14:31:09.000000000 +0000 @@ -16,18 +16,26 @@ # Use CFLAGS if specified in environment. : "${CFLAGS:=}" +CLEANUP_CMDS=() +cleanup() { + for cmd in "${CLEANUP_CMDS[@]}"; do + eval "$cmd" + done +} +trap cleanup EXIT + # Use TESTDATA if specified in environment, else generate it. if [ -z "${TESTDATA:-}" ]; then # Generate default TESTDATA file. TESTDATA=$(mktemp -t libdeflate_testdata.XXXXXXXXXX) export TESTDATA - trap 'rm -f "$TESTDATA"' EXIT + CLEANUP_CMDS+=("rm -f '$TESTDATA'") find . '(' -name '*.c' -o -name '*.h' -o -name '*.sh' ')' \ -exec cat '{}' ';' | head -c 1000000 > "$TESTDATA" fi TMPDIR=$(mktemp -d -t libdeflate_test.XXXXXXXXX) -trap 'rm -r "$TMPDIR"' EXIT +CLEANUP_CMDS+=("rm -r '$TMPDIR'") MAKE="make -j$(getconf _NPROCESSORS_ONLN)" @@ -35,12 +43,14 @@ ARCH=$(uname -m) -# Allow setting DISABLE_ASAN=1 in the environment to disable the ASAN tests. -if [ "${DISABLE_ASAN:-}" = "1" ]; then - DISABLE_ASAN=true -else - DISABLE_ASAN=false -fi +for skip in SKIP_FREESTANDING SKIP_VALGRIND SKIP_UBSAN SKIP_ASAN SKIP_CFI \ + SKIP_SHARED_LIB; do + if [ "${!skip:-}" = "1" ]; then + eval $skip=true + else + eval $skip=false + fi +done ############################################################################### @@ -183,8 +193,12 @@ do_run_tests() { build_and_run_tests "$@" if [ "${1:-}" != "--quick" ]; then - build_and_run_tests FREESTANDING=1 - verify_freestanding_build + if $SKIP_FREESTANDING; then + log "Skipping freestanding build tests due to SKIP_FREESTANDING=1" + else + build_and_run_tests FREESTANDING=1 + verify_freestanding_build + fi fi gzip_tests "$@" } @@ -204,6 +218,10 @@ } test_use_shared_lib() { + if $SKIP_SHARED_LIB; then + log "Skipping USE_SHARED_LIB=1 tests due to SKIP_SHARED_LIB=1" + return + fi log "Testing USE_SHARED_LIB=1" $MAKE gzip > /dev/null if ldd gzip | grep -q 'libdeflate.so'; then @@ -256,7 +274,9 @@ # Need valgrind 3.9.0 for '--errors-for-leak-kinds=all' # Need valgrind 3.12.0 for armv8 crypto and crc instructions - if valgrind_version_at_least 3.12.0; then + if $SKIP_VALGRIND; then + log "Skipping valgrind tests due to SKIP_VALGRIND=1" + elif valgrind_version_at_least 3.12.0; then begin "Running tests with Valgrind" WRAPPER="valgrind --quiet --error-exitcode=100 --leak-check=full --errors-for-leak-kinds=all" \ do_run_tests --quick @@ -264,7 +284,9 @@ fi cflags=("-fsanitize=undefined" "-fno-sanitize-recover=undefined") - if cflags_supported "${cflags[@]}"; then + if $SKIP_UBSAN; then + log "Skipping UBSAN tests due to SKIP_UBSAN=1" + elif cflags_supported "${cflags[@]}"; then begin "Running tests with UBSAN" CFLAGS="$CFLAGS ${cflags[*]}" do_run_tests --quick end @@ -273,8 +295,8 @@ fi cflags=("-fsanitize=address" "-fno-sanitize-recover=address") - if $DISABLE_ASAN; then - log "Skipping ASAN tests because DISABLE_ASAN=1 was set" + if $SKIP_ASAN; then + log "Skipping ASAN tests due to SKIP_ASAN=1" elif cflags_supported "${cflags[@]}"; then begin "Running tests with ASAN" CFLAGS="$CFLAGS ${cflags[*]}" do_run_tests --quick @@ -285,9 +307,11 @@ cflags=("-fsanitize=cfi" "-fno-sanitize-recover=cfi" "-flto" "-fvisibility=hidden") - if cflags_supported "${cflags[@]}"; then + if $SKIP_CFI; then + log "Skipping CFI tests due to SKIP_CFI=1" + elif cflags_supported "${cflags[@]}"; then begin "Running tests with CFI" - CFLAGS="$CFLAGS ${cflags[*]}" do_run_tests --quick + CFLAGS="$CFLAGS ${cflags[*]}" AR=llvm-ar do_run_tests --quick end else log "Skipping CFI tests because compiler ($CC_VERSION) doesn't support CFI" diff -Nru libdeflate-1.7/.travis.yml libdeflate-1.8/.travis.yml --- libdeflate-1.7/.travis.yml 2020-11-10 03:29:50.000000000 +0000 +++ libdeflate-1.8/.travis.yml 1970-01-01 00:00:00.000000000 +0000 @@ -1,113 +0,0 @@ -language: c -os: linux -dist: focal -compiler: - - gcc - - clang -env: - global: - - CFLAGS=-Werror -arch: - - amd64 - - arm64 - - ppc64le - - s390x -before_install: sudo apt-get install -y libz-dev valgrind -script: scripts/run_tests.sh - -# Additional jobs not generated by the above build matrix: -jobs: - include: - # Jobs to test older Ubuntu distros. Ideally these would be generated by - # the build matrix, but Travis CI doesn't support expanding by 'dist'... - - - dist: precise - compiler: gcc - arch: amd64 - - - dist: precise - compiler: clang - arch: amd64 - - - dist: xenial - compiler: gcc - arch: amd64 - - - dist: xenial - compiler: clang - arch: amd64 - - - dist: xenial - compiler: gcc - arch: arm64 - # Ignore incorrect -Wstrict-aliasing warning in adler32_neon_chunk(), - # not seen with any other compiler versions. - env: CFLAGS="-Wno-strict-aliasing -Werror" - - - dist: xenial - compiler: clang - arch: arm64 - # ASAN fails with internal error - env: DISABLE_ASAN=1 - - - dist: xenial - compiler: gcc - arch: ppc64le - # ASAN fails with internal error - env: DISABLE_ASAN=1 - - - dist: xenial - compiler: clang - arch: ppc64le - # ASAN fails with internal error - env: DISABLE_ASAN=1 - - - dist: xenial - compiler: gcc - arch: s390x - - - dist: xenial - compiler: clang - arch: s390x - - # Other Linux jobs - - - name: clang static analyzer - before_install: sudo apt-get install -y clang - script: make scan-build - - - name: shellcheck - before_install: sudo apt-get install -y shellcheck - script: make shellcheck - - - name: Cross compile for Windows - before_install: - - sudo apt-get install -y gcc-mingw-w64-x86-64 libz-mingw-w64-dev - script: - - make CC=i686-w64-mingw32-gcc all test_programs - - make CC=x86_64-w64-mingw32-gcc all test_programs - - # Non-Linux jobs - - - name: macOS, xcode11 - os: osx - osx_image: xcode11 - before_install: - script: make all check - - - name: macOS, xcode9.4 - os: osx - osx_image: xcode9.4 - before_install: - script: make all check - - - name: macOS, xcode7.3 - os: osx - osx_image: xcode7.3 - before_install: - script: make all check - - - name: Windows, MinGW - os: windows - before_install: - script: mingw32-make all check