From 4ab1066ade3d7c441f573e773cf85582c6bf70a0 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 15:58:54 +0200 Subject: [PATCH 1/2] ext: Add libdeflate's deflate encoder Only the compressor (deflate_compress.c, utils.c and their headers) from libdeflate 1.7, the version LunaMoo/maxcso vendors, for writing CSO files. Built as part of Core on every platform. MIT license. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CCxekYqPnzNr5d8Tp1q7aZ --- Core/CMakeLists.txt | 2 +- Core/Core.vcxproj | 15 + Core/Core.vcxproj.filters | 48 + UWP/CoreUWP/CoreUWP.vcxproj | 19 + UWP/CoreUWP/CoreUWP.vcxproj.filters | 15 + android/jni/Android.mk | 2 + ext/CMakeLists.txt | 6 + ext/libdeflate/COPYING | 21 + ext/libdeflate/README.ppsspp.md | 6 + ext/libdeflate/common/common_defs.h | 334 +++ ext/libdeflate/common/compiler_gcc.h | 185 ++ ext/libdeflate/common/compiler_msc.h | 80 + ext/libdeflate/lib/arm/matchfinder_impl.h | 81 + ext/libdeflate/lib/bt_matchfinder.h | 359 +++ ext/libdeflate/lib/deflate_compress.c | 2854 +++++++++++++++++++++ ext/libdeflate/lib/deflate_compress.h | 13 + ext/libdeflate/lib/deflate_constants.h | 66 + ext/libdeflate/lib/hc_matchfinder.h | 407 +++ ext/libdeflate/lib/lib_common.h | 67 + ext/libdeflate/lib/matchfinder_common.h | 176 ++ ext/libdeflate/lib/unaligned.h | 228 ++ ext/libdeflate/lib/utils.c | 138 + ext/libdeflate/lib/x86/matchfinder_impl.h | 117 + ext/libdeflate/libdeflate.h | 366 +++ libretro/Makefile.common | 2 + 25 files changed, 5606 insertions(+), 1 deletion(-) create mode 100644 ext/libdeflate/COPYING create mode 100644 ext/libdeflate/README.ppsspp.md create mode 100644 ext/libdeflate/common/common_defs.h create mode 100644 ext/libdeflate/common/compiler_gcc.h create mode 100644 ext/libdeflate/common/compiler_msc.h create mode 100644 ext/libdeflate/lib/arm/matchfinder_impl.h create mode 100644 ext/libdeflate/lib/bt_matchfinder.h create mode 100644 ext/libdeflate/lib/deflate_compress.c create mode 100644 ext/libdeflate/lib/deflate_compress.h create mode 100644 ext/libdeflate/lib/deflate_constants.h create mode 100644 ext/libdeflate/lib/hc_matchfinder.h create mode 100644 ext/libdeflate/lib/lib_common.h create mode 100644 ext/libdeflate/lib/matchfinder_common.h create mode 100644 ext/libdeflate/lib/unaligned.h create mode 100644 ext/libdeflate/lib/utils.c create mode 100644 ext/libdeflate/lib/x86/matchfinder_impl.h create mode 100644 ext/libdeflate/libdeflate.h diff --git a/Core/CMakeLists.txt b/Core/CMakeLists.txt index 4514429bfc..15bc787467 100644 --- a/Core/CMakeLists.txt +++ b/Core/CMakeLists.txt @@ -713,7 +713,7 @@ endif() list(APPEND CoreExtraLibs armips) -target_link_libraries(Core GPU Common chdr kirk cityhash sfmt19937 xbrz xxhash rcheevos at3_standalone lua +target_link_libraries(Core GPU Common chdr kirk cityhash sfmt19937 xbrz xxhash libdeflate rcheevos at3_standalone lua aemu_postoffice_client ${CoreExtraLibs} ${CMAKE_DL_LIBS}) # Winsock diff --git a/Core/Core.vcxproj b/Core/Core.vcxproj index aaadb94767..02835fcd16 100644 --- a/Core/Core.vcxproj +++ b/Core/Core.vcxproj @@ -738,6 +738,8 @@ + + MaxSpeed true @@ -1060,6 +1062,19 @@ + + + + + + + + + + + + + diff --git a/Core/Core.vcxproj.filters b/Core/Core.vcxproj.filters index 5c5604c4e8..bb4a8a3126 100644 --- a/Core/Core.vcxproj.filters +++ b/Core/Core.vcxproj.filters @@ -82,6 +82,9 @@ {6c6e0188-fabf-4203-aaa2-896c42be2c20} + + {5b0c7e1a-3d2f-4c8e-9a61-d4f0b2e7c913} + {0201ff27-4fa8-4cf3-bbb9-accfdc0c7046} @@ -502,6 +505,12 @@ MIPS + + Ext\libdeflate + + + Ext\libdeflate + Ext @@ -1779,6 +1788,45 @@ Ext + + Ext\libdeflate + + + Ext\libdeflate + + + Ext\libdeflate + + + Ext\libdeflate + + + Ext\libdeflate + + + Ext\libdeflate + + + Ext\libdeflate + + + Ext\libdeflate + + + Ext\libdeflate + + + Ext\libdeflate + + + Ext\libdeflate + + + Ext\libdeflate + + + Ext\libdeflate + Ext diff --git a/UWP/CoreUWP/CoreUWP.vcxproj b/UWP/CoreUWP/CoreUWP.vcxproj index 6cfdf5ffa6..f6efb8fab5 100644 --- a/UWP/CoreUWP/CoreUWP.vcxproj +++ b/UWP/CoreUWP/CoreUWP.vcxproj @@ -366,6 +366,19 @@ + + + + + + + + + + + + + @@ -1016,6 +1029,12 @@ NotUsing + + NotUsing + + + NotUsing + NotUsing diff --git a/UWP/CoreUWP/CoreUWP.vcxproj.filters b/UWP/CoreUWP/CoreUWP.vcxproj.filters index a6a3bde002..865efcbbfd 100644 --- a/UWP/CoreUWP/CoreUWP.vcxproj.filters +++ b/UWP/CoreUWP/CoreUWP.vcxproj.filters @@ -397,6 +397,8 @@ + + @@ -681,6 +683,19 @@ + + + + + + + + + + + + + diff --git a/android/jni/Android.mk b/android/jni/Android.mk index ead35d18bc..5241de86a4 100644 --- a/android/jni/Android.mk +++ b/android/jni/Android.mk @@ -552,6 +552,8 @@ EXEC_AND_LIB_FILES := \ $(ARCH_FILES) \ $(GPU_VULKAN_FILES) \ $(SRC)/ext/xxhash.c \ + $(SRC)/ext/libdeflate/lib/deflate_compress.c \ + $(SRC)/ext/libdeflate/lib/utils.c \ $(SRC)/Core/MIPS/MIPS.cpp.arm \ $(SRC)/Core/MIPS/MIPSAnalyst.cpp \ $(SRC)/Core/MIPS/MIPSDis.cpp \ diff --git a/ext/CMakeLists.txt b/ext/CMakeLists.txt index b8607ef950..d0d96f0eb8 100644 --- a/ext/CMakeLists.txt +++ b/ext/CMakeLists.txt @@ -51,6 +51,12 @@ add_library(xxhash STATIC xxhash.h ) +# Only the deflate encoder, for writing CSO files (Core/FileSystems/CSOCompress.cpp). +add_library(libdeflate STATIC + libdeflate/lib/deflate_compress.c + libdeflate/lib/utils.c +) + set(aemu_postoffice_client_obj aemu_postoffice/client/postoffice.c aemu_postoffice/client/postoffice_mem_stdc.c diff --git a/ext/libdeflate/COPYING b/ext/libdeflate/COPYING new file mode 100644 index 0000000000..1f1b81cd5a --- /dev/null +++ b/ext/libdeflate/COPYING @@ -0,0 +1,21 @@ +Copyright 2016 Eric Biggers + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation files +(the "Software"), to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, +and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ext/libdeflate/README.ppsspp.md b/ext/libdeflate/README.ppsspp.md new file mode 100644 index 0000000000..90979045d2 --- /dev/null +++ b/ext/libdeflate/README.ppsspp.md @@ -0,0 +1,6 @@ +libdeflate 1.7's deflate compressor (https://github.com/ebiggers/libdeflate), used to write CSO +files in Core/FileSystems/CSOCompress.cpp. It's the version https://github.com/LunaMoo/maxcso +vendors, so CSOs come out byte-identical to maxcso's. Only deflate_compress.c and utils.c and +their headers are here. One change: lib/deflate_compress.c and lib/utils.c include +"../libdeflate.h" instead of "libdeflate.h", so no extra include path is needed. MIT license, +see COPYING. diff --git a/ext/libdeflate/common/common_defs.h b/ext/libdeflate/common/common_defs.h new file mode 100644 index 0000000000..d56c5cf9b4 --- /dev/null +++ b/ext/libdeflate/common/common_defs.h @@ -0,0 +1,334 @@ +/* + * common_defs.h + * + * Copyright 2016 Eric Biggers + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef COMMON_COMMON_DEFS_H +#define COMMON_COMMON_DEFS_H + +#ifdef __GNUC__ +# include "compiler_gcc.h" +#elif defined(_MSC_VER) +# include "compiler_msc.h" +#else +# pragma message("Unrecognized compiler. Please add a header file for your compiler. Compilation will proceed, but performance may suffer!") +#endif + +/* ========================================================================== */ +/* Type definitions */ +/* ========================================================================== */ + +#include /* size_t */ + +#ifndef __bool_true_false_are_defined +# include /* bool */ +#endif + +/* Fixed-width integer types */ +#include +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; +typedef int8_t s8; +typedef int16_t s16; +typedef int32_t s32; +typedef int64_t s64; + +/* + * Word type of the target architecture. Use 'size_t' instead of 'unsigned + * long' to account for platforms such as Windows that use 32-bit 'unsigned + * long' on 64-bit architectures. + */ +typedef size_t machine_word_t; + +/* Number of bytes in a word */ +#define WORDBYTES ((int)sizeof(machine_word_t)) + +/* Number of bits in a word */ +#define WORDBITS (8 * WORDBYTES) + +/* ========================================================================== */ +/* Optional compiler features */ +/* ========================================================================== */ + +/* LIBEXPORT - export a function from a shared library */ +#ifndef LIBEXPORT +# define LIBEXPORT +#endif + +/* inline - suggest that a function be inlined */ +#ifndef inline +# define inline +#endif + +/* forceinline - force a function to be inlined, if possible */ +#ifndef forceinline +# define forceinline inline +#endif + +/* restrict - annotate a non-aliased pointer */ +#ifndef restrict +# define restrict +#endif + +/* likely(expr) - hint that an expression is usually true */ +#ifndef likely +# define likely(expr) (expr) +#endif + +/* unlikely(expr) - hint that an expression is usually false */ +#ifndef unlikely +# define unlikely(expr) (expr) +#endif + +/* prefetchr(addr) - prefetch into L1 cache for read */ +#ifndef prefetchr +# define prefetchr(addr) +#endif + +/* prefetchw(addr) - prefetch into L1 cache for write */ +#ifndef prefetchw +# define prefetchw(addr) +#endif + +/* Does the compiler support the 'target' function attribute? */ +#ifndef COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE +# define COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE 0 +#endif + +/* Which targets are supported with the 'target' function attribute? */ +#ifndef COMPILER_SUPPORTS_BMI2_TARGET +# define COMPILER_SUPPORTS_BMI2_TARGET 0 +#endif +#ifndef COMPILER_SUPPORTS_AVX_TARGET +# define COMPILER_SUPPORTS_AVX_TARGET 0 +#endif +#ifndef COMPILER_SUPPORTS_AVX512BW_TARGET +# define COMPILER_SUPPORTS_AVX512BW_TARGET 0 +#endif + +/* + * Which targets are supported with the 'target' function attribute and have + * intrinsics that work within 'target'-ed functions? + */ +#ifndef COMPILER_SUPPORTS_SSE2_TARGET_INTRINSICS +# define COMPILER_SUPPORTS_SSE2_TARGET_INTRINSICS 0 +#endif +#ifndef COMPILER_SUPPORTS_PCLMUL_TARGET_INTRINSICS +# define COMPILER_SUPPORTS_PCLMUL_TARGET_INTRINSICS 0 +#endif +#ifndef COMPILER_SUPPORTS_AVX2_TARGET_INTRINSICS +# define COMPILER_SUPPORTS_AVX2_TARGET_INTRINSICS 0 +#endif +#ifndef COMPILER_SUPPORTS_AVX512BW_TARGET_INTRINSICS +# define COMPILER_SUPPORTS_AVX512BW_TARGET_INTRINSICS 0 +#endif +#ifndef COMPILER_SUPPORTS_NEON_TARGET_INTRINSICS +# define COMPILER_SUPPORTS_NEON_TARGET_INTRINSICS 0 +#endif +#ifndef COMPILER_SUPPORTS_PMULL_TARGET_INTRINSICS +# define COMPILER_SUPPORTS_PMULL_TARGET_INTRINSICS 0 +#endif +#ifndef COMPILER_SUPPORTS_CRC32_TARGET_INTRINSICS +# define COMPILER_SUPPORTS_CRC32_TARGET_INTRINSICS 0 +#endif + +/* _aligned_attribute(n) - declare that the annotated variable, or variables of + * the annotated type, are to be aligned on n-byte boundaries */ +#ifndef _aligned_attribute +#endif + +/* ========================================================================== */ +/* Miscellaneous macros */ +/* ========================================================================== */ + +#define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0])) +#define MIN(a, b) ((a) <= (b) ? (a) : (b)) +#define MAX(a, b) ((a) >= (b) ? (a) : (b)) +#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) +#define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)])) +#define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1)) + +/* ========================================================================== */ +/* Endianness handling */ +/* ========================================================================== */ + +/* + * CPU_IS_LITTLE_ENDIAN() - a macro which evaluates to 1 if the CPU is little + * endian or 0 if it is big endian. The macro should be defined in a way such + * that the compiler can evaluate it at compilation time. If not defined, a + * fallback is used. + */ +#ifndef CPU_IS_LITTLE_ENDIAN +static forceinline int CPU_IS_LITTLE_ENDIAN(void) +{ + union { + unsigned int v; + unsigned char b; + } u; + u.v = 1; + return u.b; +} +#endif + +/* bswap16(n) - swap the bytes of a 16-bit integer */ +#ifndef bswap16 +static forceinline u16 bswap16(u16 n) +{ + return (n << 8) | (n >> 8); +} +#endif + +/* bswap32(n) - swap the bytes of a 32-bit integer */ +#ifndef bswap32 +static forceinline u32 bswap32(u32 n) +{ + return ((n & 0x000000FF) << 24) | + ((n & 0x0000FF00) << 8) | + ((n & 0x00FF0000) >> 8) | + ((n & 0xFF000000) >> 24); +} +#endif + +/* bswap64(n) - swap the bytes of a 64-bit integer */ +#ifndef bswap64 +static forceinline u64 bswap64(u64 n) +{ + return ((n & 0x00000000000000FF) << 56) | + ((n & 0x000000000000FF00) << 40) | + ((n & 0x0000000000FF0000) << 24) | + ((n & 0x00000000FF000000) << 8) | + ((n & 0x000000FF00000000) >> 8) | + ((n & 0x0000FF0000000000) >> 24) | + ((n & 0x00FF000000000000) >> 40) | + ((n & 0xFF00000000000000) >> 56); +} +#endif + +#define le16_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? (n) : bswap16(n)) +#define le32_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? (n) : bswap32(n)) +#define le64_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? (n) : bswap64(n)) +#define be16_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? bswap16(n) : (n)) +#define be32_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? bswap32(n) : (n)) +#define be64_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? bswap64(n) : (n)) + +/* ========================================================================== */ +/* Unaligned memory accesses */ +/* ========================================================================== */ + +/* + * UNALIGNED_ACCESS_IS_FAST should be defined to 1 if unaligned memory accesses + * can be performed efficiently on the target platform. + */ +#ifndef UNALIGNED_ACCESS_IS_FAST +# define UNALIGNED_ACCESS_IS_FAST 0 +#endif + +/* ========================================================================== */ +/* Bit scan functions */ +/* ========================================================================== */ + +/* + * Bit Scan Reverse (BSR) - find the 0-based index (relative to the least + * significant end) of the *most* significant 1 bit in the input value. The + * input value must be nonzero! + */ + +#ifndef bsr32 +static forceinline unsigned +bsr32(u32 n) +{ + unsigned i = 0; + while ((n >>= 1) != 0) + i++; + return i; +} +#endif + +#ifndef bsr64 +static forceinline unsigned +bsr64(u64 n) +{ + unsigned i = 0; + while ((n >>= 1) != 0) + i++; + return i; +} +#endif + +static forceinline unsigned +bsrw(machine_word_t n) +{ + STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64); + if (WORDBITS == 32) + return bsr32(n); + else + return bsr64(n); +} + +/* + * Bit Scan Forward (BSF) - find the 0-based index (relative to the least + * significant end) of the *least* significant 1 bit in the input value. The + * input value must be nonzero! + */ + +#ifndef bsf32 +static forceinline unsigned +bsf32(u32 n) +{ + unsigned i = 0; + while ((n & 1) == 0) { + i++; + n >>= 1; + } + return i; +} +#endif + +#ifndef bsf64 +static forceinline unsigned +bsf64(u64 n) +{ + unsigned i = 0; + while ((n & 1) == 0) { + i++; + n >>= 1; + } + return i; +} +#endif + +static forceinline unsigned +bsfw(machine_word_t n) +{ + STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64); + if (WORDBITS == 32) + return bsf32(n); + else + return bsf64(n); +} + +#endif /* COMMON_COMMON_DEFS_H */ diff --git a/ext/libdeflate/common/compiler_gcc.h b/ext/libdeflate/common/compiler_gcc.h new file mode 100644 index 0000000000..82599469a6 --- /dev/null +++ b/ext/libdeflate/common/compiler_gcc.h @@ -0,0 +1,185 @@ +/* + * compiler_gcc.h - definitions for the GNU C Compiler. This also handles clang + * and the Intel C Compiler (icc). + * + * TODO: icc is not well tested, so some things are currently disabled even + * though they maybe can be enabled on some icc versions. + */ + +#if !defined(__clang__) && !defined(__INTEL_COMPILER) +# define GCC_PREREQ(major, minor) \ + (__GNUC__ > (major) || \ + (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor))) +#else +# define GCC_PREREQ(major, minor) 0 +#endif + +/* Note: only check the clang version when absolutely necessary! + * "Vendors" such as Apple can use different version numbers. */ +#ifdef __clang__ +# ifdef __apple_build_version__ +# define CLANG_PREREQ(major, minor, apple_version) \ + (__apple_build_version__ >= (apple_version)) +# else +# define CLANG_PREREQ(major, minor, apple_version) \ + (__clang_major__ > (major) || \ + (__clang_major__ == (major) && __clang_minor__ >= (minor))) +# endif +#else +# define CLANG_PREREQ(major, minor, apple_version) 0 +#endif + +#ifndef __has_attribute +# define __has_attribute(attribute) 0 +#endif +#ifndef __has_feature +# define __has_feature(feature) 0 +#endif +#ifndef __has_builtin +# define __has_builtin(builtin) 0 +#endif + +#ifdef _WIN32 +# define LIBEXPORT __declspec(dllexport) +#else +# define LIBEXPORT __attribute__((visibility("default"))) +#endif + +#define inline inline +#define forceinline inline __attribute__((always_inline)) +#define restrict __restrict__ +#define likely(expr) __builtin_expect(!!(expr), 1) +#define unlikely(expr) __builtin_expect(!!(expr), 0) +#define prefetchr(addr) __builtin_prefetch((addr), 0) +#define prefetchw(addr) __builtin_prefetch((addr), 1) +#define _aligned_attribute(n) __attribute__((aligned(n))) + +#define COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE \ + (GCC_PREREQ(4, 4) || __has_attribute(target)) + +#if COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE + +# if defined(__i386__) || defined(__x86_64__) + +# define COMPILER_SUPPORTS_PCLMUL_TARGET \ + (GCC_PREREQ(4, 4) || __has_builtin(__builtin_ia32_pclmulqdq128)) + +# define COMPILER_SUPPORTS_AVX_TARGET \ + (GCC_PREREQ(4, 6) || __has_builtin(__builtin_ia32_maxps256)) + +# define COMPILER_SUPPORTS_BMI2_TARGET \ + (GCC_PREREQ(4, 7) || __has_builtin(__builtin_ia32_pdep_di)) + +# define COMPILER_SUPPORTS_AVX2_TARGET \ + (GCC_PREREQ(4, 7) || __has_builtin(__builtin_ia32_psadbw256)) + +# define COMPILER_SUPPORTS_AVX512BW_TARGET \ + (GCC_PREREQ(5, 1) || __has_builtin(__builtin_ia32_psadbw512)) + + /* + * Prior to gcc 4.9 (r200349) and clang 3.8 (r239883), x86 intrinsics + * not available in the main target could not be used in 'target' + * attribute functions. Unfortunately clang has no feature test macro + * for this so we have to check its version. + */ +# if GCC_PREREQ(4, 9) || CLANG_PREREQ(3, 8, 7030000) +# define COMPILER_SUPPORTS_SSE2_TARGET_INTRINSICS 1 +# define COMPILER_SUPPORTS_PCLMUL_TARGET_INTRINSICS \ + COMPILER_SUPPORTS_PCLMUL_TARGET +# define COMPILER_SUPPORTS_AVX2_TARGET_INTRINSICS \ + COMPILER_SUPPORTS_AVX2_TARGET +# define COMPILER_SUPPORTS_AVX512BW_TARGET_INTRINSICS \ + COMPILER_SUPPORTS_AVX512BW_TARGET +# endif + +# elif defined(__arm__) || defined(__aarch64__) + + /* + * Determine whether NEON and crypto intrinsics are supported. + * + * With gcc prior to 6.1, (r230411 for arm32, r226563 for arm64), neither + * was available unless enabled in the main target. + * + * But even after that, to include (which contains both the + * basic NEON intrinsics and the crypto intrinsics) the main target still + * needs to have: + * - gcc: hardware floating point support + * - clang: NEON support (but not necessarily crypto support) + */ +# if (GCC_PREREQ(6, 1) && defined(__ARM_FP)) || \ + (defined(__clang__) && defined(__ARM_NEON)) +# define COMPILER_SUPPORTS_NEON_TARGET_INTRINSICS 1 + /* + * The crypto intrinsics are broken on arm32 with clang, even when using + * -mfpu=crypto-neon-fp-armv8, because clang's puts them + * behind __aarch64__. Undefine __ARM_FEATURE_CRYPTO in that case... + */ +# if defined(__clang__) && defined(__arm__) +# undef __ARM_FEATURE_CRYPTO +# elif __has_builtin(__builtin_neon_vmull_p64) || !defined(__clang__) +# define COMPILER_SUPPORTS_PMULL_TARGET_INTRINSICS 1 +# endif +# endif + + /* + * Determine whether CRC32 intrinsics are supported. + * + * With gcc r274827 or later (gcc 10.1+, 9.3+, or 8.4+), or with clang, + * they work as expected. (Well, not quite. There's still a bug, but we + * have to work around it later when including arm_acle.h.) + */ +# if GCC_PREREQ(10, 1) || \ + (GCC_PREREQ(9, 3) && !GCC_PREREQ(10, 0)) || \ + (GCC_PREREQ(8, 4) && !GCC_PREREQ(9, 0)) || \ + (defined(__clang__) && __has_builtin(__builtin_arm_crc32b)) +# define COMPILER_SUPPORTS_CRC32_TARGET_INTRINSICS 1 +# endif + +# endif /* __arm__ || __aarch64__ */ + +#endif /* COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE */ + +/* + * Prior to gcc 5.1 and clang 3.9, emmintrin.h only defined vectors of signed + * integers (e.g. __v4si), not vectors of unsigned integers (e.g. __v4su). But + * we need the unsigned ones in order to avoid signed integer overflow, which is + * undefined behavior. Add the missing definitions for the unsigned ones if + * needed. + */ +#if (GCC_PREREQ(4, 0) && !GCC_PREREQ(5, 1)) || \ + (defined(__clang__) && !CLANG_PREREQ(3, 9, 8020000)) +typedef unsigned long long __v2du __attribute__((__vector_size__(16))); +typedef unsigned int __v4su __attribute__((__vector_size__(16))); +typedef unsigned short __v8hu __attribute__((__vector_size__(16))); +typedef unsigned char __v16qu __attribute__((__vector_size__(16))); +typedef unsigned long long __v4du __attribute__((__vector_size__(32))); +typedef unsigned int __v8su __attribute__((__vector_size__(32))); +typedef unsigned short __v16hu __attribute__((__vector_size__(32))); +typedef unsigned char __v32qu __attribute__((__vector_size__(32))); +#endif + +/* Newer gcc supports __BYTE_ORDER__. Older gcc doesn't. */ +#ifdef __BYTE_ORDER__ +# define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +#endif + +#if GCC_PREREQ(4, 8) || __has_builtin(__builtin_bswap16) +# define bswap16 __builtin_bswap16 +#endif + +#if GCC_PREREQ(4, 3) || __has_builtin(__builtin_bswap32) +# define bswap32 __builtin_bswap32 +#endif + +#if GCC_PREREQ(4, 3) || __has_builtin(__builtin_bswap64) +# define bswap64 __builtin_bswap64 +#endif + +#if defined(__x86_64__) || defined(__i386__) || defined(__ARM_FEATURE_UNALIGNED) || defined(__powerpc64__) +# define UNALIGNED_ACCESS_IS_FAST 1 +#endif + +#define bsr32(n) (31 - __builtin_clz(n)) +#define bsr64(n) (63 - __builtin_clzll(n)) +#define bsf32(n) __builtin_ctz(n) +#define bsf64(n) __builtin_ctzll(n) diff --git a/ext/libdeflate/common/compiler_msc.h b/ext/libdeflate/common/compiler_msc.h new file mode 100644 index 0000000000..18cfa128f1 --- /dev/null +++ b/ext/libdeflate/common/compiler_msc.h @@ -0,0 +1,80 @@ +/* + * compiler_msc.h - definitions for the Microsoft C Compiler + */ + +#include +#include /* for _byteswap_*() */ + +#define LIBEXPORT __declspec(dllexport) + +/* + * Old versions (e.g. VS2010) of MSC don't have the C99 header stdbool.h. + * Beware: the below replacement isn't fully standard, since normally any value + * != 0 should be implicitly cast to a bool with value 1... but that doesn't + * happen if bool is really just an 'int'. + */ +typedef int bool; +#define true 1 +#define false 0 +#define __bool_true_false_are_defined 1 + +/* Define ssize_t */ +#ifdef _WIN64 +typedef long long ssize_t; +#else +typedef int ssize_t; +#endif + +/* Assume a little endian architecture with fast unaligned access */ +#define CPU_IS_LITTLE_ENDIAN() 1 +#define UNALIGNED_ACCESS_IS_FAST 1 + +/* __restrict has nonstandard behavior; don't use it */ +#define restrict + +/* ... but we can use __inline and __forceinline */ +#define inline __inline +#define forceinline __forceinline + +/* Byte swap functions */ +#define bswap16 _byteswap_ushort +#define bswap32 _byteswap_ulong +#define bswap64 _byteswap_uint64 + +/* Bit scan functions (32-bit) */ + +static forceinline unsigned +bsr32(uint32_t n) +{ + _BitScanReverse(&n, n); + return n; +} +#define bsr32 bsr32 + +static forceinline unsigned +bsf32(uint32_t n) +{ + _BitScanForward(&n, n); + return n; +} +#define bsf32 bsf32 + +#ifdef _M_X64 /* Bit scan functions (64-bit) */ + +static forceinline unsigned +bsr64(uint64_t n) +{ + _BitScanReverse64(&n, n); + return n; +} +#define bsr64 bsr64 + +static forceinline unsigned +bsf64(uint64_t n) +{ + _BitScanForward64(&n, n); + return n; +} +#define bsf64 bsf64 + +#endif /* _M_X64 */ diff --git a/ext/libdeflate/lib/arm/matchfinder_impl.h b/ext/libdeflate/lib/arm/matchfinder_impl.h new file mode 100644 index 0000000000..9e711cc646 --- /dev/null +++ b/ext/libdeflate/lib/arm/matchfinder_impl.h @@ -0,0 +1,81 @@ +/* + * arm/matchfinder_impl.h - ARM implementations of matchfinder functions + * + * Copyright 2016 Eric Biggers + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifdef __ARM_NEON +# include +static forceinline void +matchfinder_init_neon(mf_pos_t *data, size_t size) +{ + int16x8_t *p = (int16x8_t *)data; + int16x8_t v = (int16x8_t) { + MATCHFINDER_INITVAL, MATCHFINDER_INITVAL, MATCHFINDER_INITVAL, + MATCHFINDER_INITVAL, MATCHFINDER_INITVAL, MATCHFINDER_INITVAL, + MATCHFINDER_INITVAL, MATCHFINDER_INITVAL, + }; + + STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0); + STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0); + STATIC_ASSERT(sizeof(mf_pos_t) == 2); + + do { + p[0] = v; + p[1] = v; + p[2] = v; + p[3] = v; + p += 4; + size -= 4 * sizeof(*p); + } while (size != 0); +} +#define matchfinder_init matchfinder_init_neon + +static forceinline void +matchfinder_rebase_neon(mf_pos_t *data, size_t size) +{ + int16x8_t *p = (int16x8_t *)data; + int16x8_t v = (int16x8_t) { + (u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE, + (u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE, + (u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE, + (u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE, + }; + + STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0); + STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0); + STATIC_ASSERT(sizeof(mf_pos_t) == 2); + + do { + p[0] = vqaddq_s16(p[0], v); + p[1] = vqaddq_s16(p[1], v); + p[2] = vqaddq_s16(p[2], v); + p[3] = vqaddq_s16(p[3], v); + p += 4; + size -= 4 * sizeof(*p); + } while (size != 0); +} +#define matchfinder_rebase matchfinder_rebase_neon + +#endif /* __ARM_NEON */ diff --git a/ext/libdeflate/lib/bt_matchfinder.h b/ext/libdeflate/lib/bt_matchfinder.h new file mode 100644 index 0000000000..546a19d31a --- /dev/null +++ b/ext/libdeflate/lib/bt_matchfinder.h @@ -0,0 +1,359 @@ +/* + * bt_matchfinder.h - Lempel-Ziv matchfinding with a hash table of binary trees + * + * Originally public domain; changes after 2016-09-07 are copyrighted. + * + * Copyright 2016 Eric Biggers + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * ---------------------------------------------------------------------------- + * + * This is a Binary Trees (bt) based matchfinder. + * + * The main data structure is a hash table where each hash bucket contains a + * binary tree of sequences whose first 4 bytes share the same hash code. Each + * sequence is identified by its starting position in the input buffer. Each + * binary tree is always sorted such that each left child represents a sequence + * lexicographically lesser than its parent and each right child represents a + * sequence lexicographically greater than its parent. + * + * The algorithm processes the input buffer sequentially. At each byte + * position, the hash code of the first 4 bytes of the sequence beginning at + * that position (the sequence being matched against) is computed. This + * identifies the hash bucket to use for that position. Then, a new binary tree + * node is created to represent the current sequence. Then, in a single tree + * traversal, the hash bucket's binary tree is searched for matches and is + * re-rooted at the new node. + * + * Compared to the simpler algorithm that uses linked lists instead of binary + * trees (see hc_matchfinder.h), the binary tree version gains more information + * at each node visitation. Ideally, the binary tree version will examine only + * 'log(n)' nodes to find the same matches that the linked list version will + * find by examining 'n' nodes. In addition, the binary tree version can + * examine fewer bytes at each node by taking advantage of the common prefixes + * that result from the sort order, whereas the linked list version may have to + * examine up to the full length of the match at each node. + * + * However, it is not always best to use the binary tree version. It requires + * nearly twice as much memory as the linked list version, and it takes time to + * keep the binary trees sorted, even at positions where the compressor does not + * need matches. Generally, when doing fast compression on small buffers, + * binary trees are the wrong approach. They are best suited for thorough + * compression and/or large buffers. + * + * ---------------------------------------------------------------------------- + */ + + +#include "matchfinder_common.h" + +#define BT_MATCHFINDER_HASH3_ORDER 16 +#define BT_MATCHFINDER_HASH3_WAYS 2 +#define BT_MATCHFINDER_HASH4_ORDER 16 + +#define BT_MATCHFINDER_TOTAL_HASH_SIZE \ + (((1UL << BT_MATCHFINDER_HASH3_ORDER) * BT_MATCHFINDER_HASH3_WAYS + \ + (1UL << BT_MATCHFINDER_HASH4_ORDER)) * sizeof(mf_pos_t)) + +/* Representation of a match found by the bt_matchfinder */ +struct lz_match { + + /* The number of bytes matched. */ + u16 length; + + /* The offset back from the current position that was matched. */ + u16 offset; +}; + +struct bt_matchfinder { + + /* The hash table for finding length 3 matches */ + mf_pos_t hash3_tab[1UL << BT_MATCHFINDER_HASH3_ORDER][BT_MATCHFINDER_HASH3_WAYS]; + + /* The hash table which contains the roots of the binary trees for + * finding length 4+ matches */ + mf_pos_t hash4_tab[1UL << BT_MATCHFINDER_HASH4_ORDER]; + + /* The child node references for the binary trees. The left and right + * children of the node for the sequence with position 'pos' are + * 'child_tab[pos * 2]' and 'child_tab[pos * 2 + 1]', respectively. */ + mf_pos_t child_tab[2UL * MATCHFINDER_WINDOW_SIZE]; + +} +#ifdef _aligned_attribute +_aligned_attribute(MATCHFINDER_MEM_ALIGNMENT) +#endif +; + +/* Prepare the matchfinder for a new input buffer. */ +static forceinline void +bt_matchfinder_init(struct bt_matchfinder *mf) +{ + STATIC_ASSERT(BT_MATCHFINDER_TOTAL_HASH_SIZE % + MATCHFINDER_SIZE_ALIGNMENT == 0); + + matchfinder_init((mf_pos_t *)mf, BT_MATCHFINDER_TOTAL_HASH_SIZE); +} + +static forceinline void +bt_matchfinder_slide_window(struct bt_matchfinder *mf) +{ + STATIC_ASSERT(sizeof(*mf) % MATCHFINDER_SIZE_ALIGNMENT == 0); + + matchfinder_rebase((mf_pos_t *)mf, sizeof(*mf)); +} + +static forceinline mf_pos_t * +bt_left_child(struct bt_matchfinder *mf, s32 node) +{ + return &mf->child_tab[2 * (node & (MATCHFINDER_WINDOW_SIZE - 1)) + 0]; +} + +static forceinline mf_pos_t * +bt_right_child(struct bt_matchfinder *mf, s32 node) +{ + return &mf->child_tab[2 * (node & (MATCHFINDER_WINDOW_SIZE - 1)) + 1]; +} + +/* The minimum permissible value of 'max_len' for bt_matchfinder_get_matches() + * and bt_matchfinder_skip_position(). There must be sufficiently many bytes + * remaining to load a 32-bit integer from the *next* position. */ +#define BT_MATCHFINDER_REQUIRED_NBYTES 5 + +/* Advance the binary tree matchfinder by one byte, optionally recording + * matches. @record_matches should be a compile-time constant. */ +static forceinline struct lz_match * +bt_matchfinder_advance_one_byte(struct bt_matchfinder * const restrict mf, + const u8 * const restrict in_base, + const ptrdiff_t cur_pos, + const u32 max_len, + const u32 nice_len, + const u32 max_search_depth, + u32 * const restrict next_hashes, + u32 * const restrict best_len_ret, + struct lz_match * restrict lz_matchptr, + const bool record_matches) +{ + const u8 *in_next = in_base + cur_pos; + u32 depth_remaining = max_search_depth; + const s32 cutoff = cur_pos - MATCHFINDER_WINDOW_SIZE; + u32 next_hashseq; + u32 hash3; + u32 hash4; + s32 cur_node; +#if BT_MATCHFINDER_HASH3_WAYS >= 2 + s32 cur_node_2; +#endif + const u8 *matchptr; + mf_pos_t *pending_lt_ptr, *pending_gt_ptr; + u32 best_lt_len, best_gt_len; + u32 len; + u32 best_len = 3; + + STATIC_ASSERT(BT_MATCHFINDER_HASH3_WAYS >= 1 && + BT_MATCHFINDER_HASH3_WAYS <= 2); + + next_hashseq = get_unaligned_le32(in_next + 1); + + hash3 = next_hashes[0]; + hash4 = next_hashes[1]; + + next_hashes[0] = lz_hash(next_hashseq & 0xFFFFFF, BT_MATCHFINDER_HASH3_ORDER); + next_hashes[1] = lz_hash(next_hashseq, BT_MATCHFINDER_HASH4_ORDER); + prefetchw(&mf->hash3_tab[next_hashes[0]]); + prefetchw(&mf->hash4_tab[next_hashes[1]]); + + cur_node = mf->hash3_tab[hash3][0]; + mf->hash3_tab[hash3][0] = cur_pos; +#if BT_MATCHFINDER_HASH3_WAYS >= 2 + cur_node_2 = mf->hash3_tab[hash3][1]; + mf->hash3_tab[hash3][1] = cur_node; +#endif + if (record_matches && cur_node > cutoff) { + u32 seq3 = load_u24_unaligned(in_next); + if (seq3 == load_u24_unaligned(&in_base[cur_node])) { + lz_matchptr->length = 3; + lz_matchptr->offset = in_next - &in_base[cur_node]; + lz_matchptr++; + } + #if BT_MATCHFINDER_HASH3_WAYS >= 2 + else if (cur_node_2 > cutoff && + seq3 == load_u24_unaligned(&in_base[cur_node_2])) + { + lz_matchptr->length = 3; + lz_matchptr->offset = in_next - &in_base[cur_node_2]; + lz_matchptr++; + } + #endif + } + + cur_node = mf->hash4_tab[hash4]; + mf->hash4_tab[hash4] = cur_pos; + + pending_lt_ptr = bt_left_child(mf, cur_pos); + pending_gt_ptr = bt_right_child(mf, cur_pos); + + if (cur_node <= cutoff) { + *pending_lt_ptr = MATCHFINDER_INITVAL; + *pending_gt_ptr = MATCHFINDER_INITVAL; + *best_len_ret = best_len; + return lz_matchptr; + } + + best_lt_len = 0; + best_gt_len = 0; + len = 0; + + for (;;) { + matchptr = &in_base[cur_node]; + + if (matchptr[len] == in_next[len]) { + len = lz_extend(in_next, matchptr, len + 1, max_len); + if (!record_matches || len > best_len) { + if (record_matches) { + best_len = len; + lz_matchptr->length = len; + lz_matchptr->offset = in_next - matchptr; + lz_matchptr++; + } + if (len >= nice_len) { + *pending_lt_ptr = *bt_left_child(mf, cur_node); + *pending_gt_ptr = *bt_right_child(mf, cur_node); + *best_len_ret = best_len; + return lz_matchptr; + } + } + } + + if (matchptr[len] < in_next[len]) { + *pending_lt_ptr = cur_node; + pending_lt_ptr = bt_right_child(mf, cur_node); + cur_node = *pending_lt_ptr; + best_lt_len = len; + if (best_gt_len < len) + len = best_gt_len; + } else { + *pending_gt_ptr = cur_node; + pending_gt_ptr = bt_left_child(mf, cur_node); + cur_node = *pending_gt_ptr; + best_gt_len = len; + if (best_lt_len < len) + len = best_lt_len; + } + + if (cur_node <= cutoff || !--depth_remaining) { + *pending_lt_ptr = MATCHFINDER_INITVAL; + *pending_gt_ptr = MATCHFINDER_INITVAL; + *best_len_ret = best_len; + return lz_matchptr; + } + } +} + +/* + * Retrieve a list of matches with the current position. + * + * @mf + * The matchfinder structure. + * @in_base + * Pointer to the next byte in the input buffer to process _at the last + * time bt_matchfinder_init() or bt_matchfinder_slide_window() was called_. + * @cur_pos + * The current position in the input buffer relative to @in_base (the + * position of the sequence being matched against). + * @max_len + * The maximum permissible match length at this position. Must be >= + * BT_MATCHFINDER_REQUIRED_NBYTES. + * @nice_len + * Stop searching if a match of at least this length is found. + * Must be <= @max_len. + * @max_search_depth + * Limit on the number of potential matches to consider. Must be >= 1. + * @next_hashes + * The precomputed hash codes for the sequence beginning at @in_next. + * These will be used and then updated with the precomputed hashcodes for + * the sequence beginning at @in_next + 1. + * @best_len_ret + * If a match of length >= 4 was found, then the length of the longest such + * match is written here; otherwise 3 is written here. (Note: this is + * redundant with the 'struct lz_match' array, but this is easier for the + * compiler to optimize when inlined and the caller immediately does a + * check against 'best_len'.) + * @lz_matchptr + * An array in which this function will record the matches. The recorded + * matches will be sorted by strictly increasing length and (non-strictly) + * increasing offset. The maximum number of matches that may be found is + * 'nice_len - 2'. + * + * The return value is a pointer to the next available slot in the @lz_matchptr + * array. (If no matches were found, this will be the same as @lz_matchptr.) + */ +static forceinline struct lz_match * +bt_matchfinder_get_matches(struct bt_matchfinder *mf, + const u8 *in_base, + ptrdiff_t cur_pos, + u32 max_len, + u32 nice_len, + u32 max_search_depth, + u32 next_hashes[2], + u32 *best_len_ret, + struct lz_match *lz_matchptr) +{ + return bt_matchfinder_advance_one_byte(mf, + in_base, + cur_pos, + max_len, + nice_len, + max_search_depth, + next_hashes, + best_len_ret, + lz_matchptr, + true); +} + +/* + * Advance the matchfinder, but don't record any matches. + * + * This is very similar to bt_matchfinder_get_matches() because both functions + * must do hashing and tree re-rooting. + */ +static forceinline void +bt_matchfinder_skip_position(struct bt_matchfinder *mf, + const u8 *in_base, + ptrdiff_t cur_pos, + u32 nice_len, + u32 max_search_depth, + u32 next_hashes[2]) +{ + u32 best_len; + bt_matchfinder_advance_one_byte(mf, + in_base, + cur_pos, + nice_len, + nice_len, + max_search_depth, + next_hashes, + &best_len, + NULL, + false); +} diff --git a/ext/libdeflate/lib/deflate_compress.c b/ext/libdeflate/lib/deflate_compress.c new file mode 100644 index 0000000000..eb1d614d3d --- /dev/null +++ b/ext/libdeflate/lib/deflate_compress.c @@ -0,0 +1,2854 @@ +/* + * deflate_compress.c - a compressor for DEFLATE + * + * Originally public domain; changes after 2016-09-07 are copyrighted. + * + * Copyright 2016 Eric Biggers + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "deflate_compress.h" +#include "deflate_constants.h" +#include "unaligned.h" + +#include "../libdeflate.h" + +/* + * By default, the near-optimal parsing algorithm is enabled at compression + * level 8 and above. The near-optimal parsing algorithm produces a compression + * ratio significantly better than the greedy and lazy algorithms implemented + * here, and also the algorithm used by zlib at level 9. However, it is slow. + */ +#define SUPPORT_NEAR_OPTIMAL_PARSING 1 + +/* + * Define to 1 to maintain the full map from match offsets to offset slots. + * This slightly speeds up translations of match offsets to offset slots, but it + * uses 32769 bytes of memory rather than the 512 bytes used by the condensed + * map. The speedup provided by the larger map is most helpful when the + * near-optimal parsing algorithm is being used. + */ +#define USE_FULL_OFFSET_SLOT_FAST SUPPORT_NEAR_OPTIMAL_PARSING + +/* + * DEFLATE uses a 32768 byte sliding window; set the matchfinder parameters + * appropriately. + */ +#define MATCHFINDER_WINDOW_ORDER 15 + +#include "hc_matchfinder.h" +#if SUPPORT_NEAR_OPTIMAL_PARSING +# include "bt_matchfinder.h" +#endif + +/* + * The compressor always chooses a block of at least MIN_BLOCK_LENGTH bytes, + * except if the last block has to be shorter. + */ +#define MIN_BLOCK_LENGTH 10000 + +/* + * The compressor attempts to end blocks after SOFT_MAX_BLOCK_LENGTH bytes, but + * the final length might be slightly longer due to matches extending beyond + * this limit. + */ +#define SOFT_MAX_BLOCK_LENGTH 300000 + +/* + * The number of observed matches or literals that represents sufficient data to + * decide whether the current block should be terminated or not. + */ +#define NUM_OBSERVATIONS_PER_BLOCK_CHECK 512 + + +#if SUPPORT_NEAR_OPTIMAL_PARSING +/* Constants specific to the near-optimal parsing algorithm */ + +/* + * The maximum number of matches the matchfinder can find at a single position. + * Since the matchfinder never finds more than one match for the same length, + * presuming one of each possible length is sufficient for an upper bound. + * (This says nothing about whether it is worthwhile to consider so many + * matches; this is just defining the worst case.) + */ +# define MAX_MATCHES_PER_POS (DEFLATE_MAX_MATCH_LEN - DEFLATE_MIN_MATCH_LEN + 1) + +/* + * The number of lz_match structures in the match cache, excluding the extra + * "overflow" entries. This value should be high enough so that nearly the + * time, all matches found in a given block can fit in the match cache. + * However, fallback behavior (immediately terminating the block) on cache + * overflow is still required. + */ +# define CACHE_LENGTH (SOFT_MAX_BLOCK_LENGTH * 5) + +#endif /* SUPPORT_NEAR_OPTIMAL_PARSING */ + +/* + * These are the compressor-side limits on the codeword lengths for each Huffman + * code. To make outputting bits slightly faster, some of these limits are + * lower than the limits defined by the DEFLATE format. This does not + * significantly affect the compression ratio, at least for the block lengths we + * use. + */ +#define MAX_LITLEN_CODEWORD_LEN 14 +#define MAX_OFFSET_CODEWORD_LEN DEFLATE_MAX_OFFSET_CODEWORD_LEN +#define MAX_PRE_CODEWORD_LEN DEFLATE_MAX_PRE_CODEWORD_LEN + +/* Table: length slot => length slot base value */ +static const unsigned deflate_length_slot_base[] = { + 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 , + 11 , 13 , 15 , 17 , 19 , 23 , 27 , 31 , + 35 , 43 , 51 , 59 , 67 , 83 , 99 , 115 , + 131 , 163 , 195 , 227 , 258 , +}; + +/* Table: length slot => number of extra length bits */ +static const u8 deflate_extra_length_bits[] = { + 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , + 1 , 1 , 1 , 1 , 2 , 2 , 2 , 2 , + 3 , 3 , 3 , 3 , 4 , 4 , 4 , 4 , + 5 , 5 , 5 , 5 , 0 , +}; + +/* Table: offset slot => offset slot base value */ +static const unsigned deflate_offset_slot_base[] = { + 1 , 2 , 3 , 4 , 5 , 7 , 9 , 13 , + 17 , 25 , 33 , 49 , 65 , 97 , 129 , 193 , + 257 , 385 , 513 , 769 , 1025 , 1537 , 2049 , 3073 , + 4097 , 6145 , 8193 , 12289 , 16385 , 24577 , +}; + +/* Table: offset slot => number of extra offset bits */ +static const u8 deflate_extra_offset_bits[] = { + 0 , 0 , 0 , 0 , 1 , 1 , 2 , 2 , + 3 , 3 , 4 , 4 , 5 , 5 , 6 , 6 , + 7 , 7 , 8 , 8 , 9 , 9 , 10 , 10 , + 11 , 11 , 12 , 12 , 13 , 13 , +}; + +/* Table: length => length slot */ +static const u8 deflate_length_slot[DEFLATE_MAX_MATCH_LEN + 1] = { + 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 12, + 12, 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, 15, 15, 16, 16, 16, 16, 16, + 16, 16, 16, 17, 17, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 18, 18, + 18, 19, 19, 19, 19, 19, 19, 19, 19, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, + 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, + 27, 27, 28, +}; + +/* The order in which precode codeword lengths are stored */ +static const u8 deflate_precode_lens_permutation[DEFLATE_NUM_PRECODE_SYMS] = { + 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 +}; + +/* Codewords for the DEFLATE Huffman codes. */ +struct deflate_codewords { + u32 litlen[DEFLATE_NUM_LITLEN_SYMS]; + u32 offset[DEFLATE_NUM_OFFSET_SYMS]; +}; + +/* Codeword lengths (in bits) for the DEFLATE Huffman codes. + * A zero length means the corresponding symbol had zero frequency. */ +struct deflate_lens { + u8 litlen[DEFLATE_NUM_LITLEN_SYMS]; + u8 offset[DEFLATE_NUM_OFFSET_SYMS]; +}; + +/* Codewords and lengths for the DEFLATE Huffman codes. */ +struct deflate_codes { + struct deflate_codewords codewords; + struct deflate_lens lens; +}; + +/* Symbol frequency counters for the DEFLATE Huffman codes. */ +struct deflate_freqs { + u32 litlen[DEFLATE_NUM_LITLEN_SYMS]; + u32 offset[DEFLATE_NUM_OFFSET_SYMS]; +}; + +#if SUPPORT_NEAR_OPTIMAL_PARSING + +/* Costs for the near-optimal parsing algorithm. */ +struct deflate_costs { + + /* The cost to output each possible literal. */ + u32 literal[DEFLATE_NUM_LITERALS]; + + /* The cost to output each possible match length. */ + u32 length[DEFLATE_MAX_MATCH_LEN + 1]; + + /* The cost to output a match offset of each possible offset slot. */ + u32 offset_slot[DEFLATE_NUM_OFFSET_SYMS]; +}; + +/* + * COST_SHIFT is a scaling factor that makes it possible to consider fractional + * bit costs. A token requiring 'n' bits to represent has cost n << COST_SHIFT. + * + * Note: this is only useful as a statistical trick for when the true costs are + * unknown. In reality, each token in DEFLATE requires a whole number of bits + * to output. + */ +#define COST_SHIFT 3 + +/* + * The NOSTAT_BITS value for a given alphabet is the number of bits assumed to + * be needed to output a symbol that was unused in the previous optimization + * pass. Assigning a default cost allows the symbol to be used in the next + * optimization pass. However, the cost should be relatively high because the + * symbol probably won't be used very many times (if at all). + */ +#define LITERAL_NOSTAT_BITS 13 +#define LENGTH_NOSTAT_BITS 13 +#define OFFSET_NOSTAT_BITS 10 + +#endif /* SUPPORT_NEAR_OPTIMAL_PARSING */ + +/* + * Represents a run of literals followed by a match or end-of-block. This + * struct is needed to temporarily store items chosen by the parser, since items + * cannot be written until all items for the block have been chosen and the + * block's Huffman codes have been computed. + */ +struct deflate_sequence { + + /* Bits 0..22: the number of literals in this run. This may be 0 and + * can be at most about SOFT_MAX_BLOCK_LENGTH. The literals are not + * stored explicitly in this structure; instead, they are read directly + * from the uncompressed data. + * + * Bits 23..31: the length of the match which follows the literals, or 0 + * if this literal run was the last in the block, so there is no match + * which follows it. */ + u32 litrunlen_and_length; + + /* If 'length' doesn't indicate end-of-block, then this is the offset of + * the match which follows the literals. */ + u16 offset; + + /* If 'length' doesn't indicate end-of-block, then this is the offset + * symbol of the match which follows the literals. */ + u8 offset_symbol; + + /* If 'length' doesn't indicate end-of-block, then this is the length + * slot of the match which follows the literals. */ + u8 length_slot; +}; + +#if SUPPORT_NEAR_OPTIMAL_PARSING + +/* + * This structure represents a byte position in the input data and a node in the + * graph of possible match/literal choices for the current block. + * + * Logically, each incoming edge to this node is labeled with a literal or a + * match that can be taken to reach this position from an earlier position; and + * each outgoing edge from this node is labeled with a literal or a match that + * can be taken to advance from this position to a later position. + * + * But these "edges" are actually stored elsewhere (in 'match_cache'). Here we + * associate with each node just two pieces of information: + * + * 'cost_to_end' is the minimum cost to reach the end of the block from + * this position. + * + * 'item' represents the literal or match that must be chosen from here to + * reach the end of the block with the minimum cost. Equivalently, this + * can be interpreted as the label of the outgoing edge on the minimum-cost + * path to the "end of block" node from this node. + */ +struct deflate_optimum_node { + + u32 cost_to_end; + + /* + * Notes on the match/literal representation used here: + * + * The low bits of 'item' are the length: 1 if this is a literal, + * or the match length if this is a match. + * + * The high bits of 'item' are the actual literal byte if this is a + * literal, or the match offset if this is a match. + */ +#define OPTIMUM_OFFSET_SHIFT 9 +#define OPTIMUM_LEN_MASK (((u32)1 << OPTIMUM_OFFSET_SHIFT) - 1) + u32 item; + +}; + +#endif /* SUPPORT_NEAR_OPTIMAL_PARSING */ + +/* Block split statistics. See "Block splitting algorithm" below. */ +#define NUM_LITERAL_OBSERVATION_TYPES 8 +#define NUM_MATCH_OBSERVATION_TYPES 2 +#define NUM_OBSERVATION_TYPES (NUM_LITERAL_OBSERVATION_TYPES + NUM_MATCH_OBSERVATION_TYPES) +struct block_split_stats { + u32 new_observations[NUM_OBSERVATION_TYPES]; + u32 observations[NUM_OBSERVATION_TYPES]; + u32 num_new_observations; + u32 num_observations; +}; + +/* The main DEFLATE compressor structure */ +struct libdeflate_compressor { + + /* Pointer to the compress() implementation chosen at allocation time */ + size_t (*impl)(struct libdeflate_compressor *, + const u8 *, size_t, u8 *, size_t); + + /* Frequency counters for the current block */ + struct deflate_freqs freqs; + + /* Dynamic Huffman codes for the current block */ + struct deflate_codes codes; + + /* Static Huffman codes */ + struct deflate_codes static_codes; + + /* Block split statistics for the currently pending block */ + struct block_split_stats split_stats; + + /* A table for fast lookups of offset slot by match offset. + * + * If the full table is being used, it is a direct mapping from offset + * to offset slot. + * + * If the condensed table is being used, the first 256 entries map + * directly to the offset slots of offsets 1 through 256. The next 256 + * entries map to the offset slots for the remaining offsets, stepping + * through the offsets with a stride of 128. This relies on the fact + * that each of the remaining offset slots contains at least 128 offsets + * and has an offset base that is a multiple of 128. */ +#if USE_FULL_OFFSET_SLOT_FAST + u8 offset_slot_fast[DEFLATE_MAX_MATCH_OFFSET + 1]; +#else + u8 offset_slot_fast[512]; +#endif + + /* The "nice" match length: if a match of this length is found, choose + * it immediately without further consideration. */ + unsigned nice_match_length; + + /* The maximum search depth: consider at most this many potential + * matches at each position. */ + unsigned max_search_depth; + + /* The compression level with which this compressor was created. */ + unsigned compression_level; + + /* Anything smaller than this we won't bother trying to compress. */ + unsigned min_size_to_compress; + + /* Temporary space for Huffman code output */ + u32 precode_freqs[DEFLATE_NUM_PRECODE_SYMS]; + u8 precode_lens[DEFLATE_NUM_PRECODE_SYMS]; + u32 precode_codewords[DEFLATE_NUM_PRECODE_SYMS]; + unsigned precode_items[DEFLATE_NUM_LITLEN_SYMS + DEFLATE_NUM_OFFSET_SYMS]; + unsigned num_litlen_syms; + unsigned num_offset_syms; + unsigned num_explicit_lens; + unsigned num_precode_items; + + union { + /* Data for greedy or lazy parsing */ + struct { + /* Hash chain matchfinder */ + struct hc_matchfinder hc_mf; + + /* The matches and literals that the parser has chosen + * for the current block. The required length of this + * array is limited by the maximum number of matches + * that can ever be chosen for a single block, plus one + * for the special entry at the end. */ + struct deflate_sequence sequences[ + DIV_ROUND_UP(SOFT_MAX_BLOCK_LENGTH, + DEFLATE_MIN_MATCH_LEN) + 1]; + } g; /* (g)reedy */ + + #if SUPPORT_NEAR_OPTIMAL_PARSING + /* Data for near-optimal parsing */ + struct { + + /* Binary tree matchfinder */ + struct bt_matchfinder bt_mf; + + /* + * Cached matches for the current block. This array + * contains the matches that were found at each position + * in the block. Specifically, for each position, there + * is a list of matches found at that position, if any, + * sorted by strictly increasing length. In addition, + * following the matches for each position, there is a + * special 'struct lz_match' whose 'length' member + * contains the number of matches found at that + * position, and whose 'offset' member contains the + * literal at that position. + * + * Note: in rare cases, there will be a very high number + * of matches in the block and this array will overflow. + * If this happens, we force the end of the current + * block. CACHE_LENGTH is the length at which we + * actually check for overflow. The extra slots beyond + * this are enough to absorb the worst case overflow, + * which occurs if starting at &match_cache[CACHE_LENGTH + * - 1], we write MAX_MATCHES_PER_POS matches and a + * match count header, then skip searching for matches + * at 'DEFLATE_MAX_MATCH_LEN - 1' positions and write + * the match count header for each. + */ + struct lz_match match_cache[CACHE_LENGTH + + MAX_MATCHES_PER_POS + + DEFLATE_MAX_MATCH_LEN - 1]; + + /* + * Array of nodes, one per position, for running the + * minimum-cost path algorithm. + * + * This array must be large enough to accommodate the + * worst-case number of nodes, which occurs if we find a + * match of length DEFLATE_MAX_MATCH_LEN at position + * SOFT_MAX_BLOCK_LENGTH - 1, producing a block of + * length SOFT_MAX_BLOCK_LENGTH - 1 + + * DEFLATE_MAX_MATCH_LEN. Add one for the end-of-block + * node. + */ + struct deflate_optimum_node optimum_nodes[SOFT_MAX_BLOCK_LENGTH - 1 + + DEFLATE_MAX_MATCH_LEN + 1]; + + /* The current cost model being used. */ + struct deflate_costs costs; + + unsigned num_optim_passes; + } n; /* (n)ear-optimal */ + #endif /* SUPPORT_NEAR_OPTIMAL_PARSING */ + + } p; /* (p)arser */ +}; + +/* + * The type for the bitbuffer variable, which temporarily holds bits that are + * being packed into bytes and written to the output buffer. For best + * performance, this should have size equal to a machine word. + */ +typedef machine_word_t bitbuf_t; +#define BITBUF_NBITS (8 * sizeof(bitbuf_t)) + +/* Can the specified number of bits always be added to 'bitbuf' after any + * pending bytes have been flushed? */ +#define CAN_BUFFER(n) ((n) <= BITBUF_NBITS - 7) + +/* + * Structure to keep track of the current state of sending bits to the + * compressed output buffer. + */ +struct deflate_output_bitstream { + + /* Bits that haven't yet been written to the output buffer. */ + bitbuf_t bitbuf; + + /* Number of bits currently held in @bitbuf. */ + unsigned bitcount; + + /* Pointer to the beginning of the output buffer. */ + u8 *begin; + + /* Pointer to the position in the output buffer at which the next byte + * should be written. */ + u8 *next; + + /* Pointer just past the end of the output buffer. */ + u8 *end; +}; + +/* + * OUTPUT_END_PADDING is the size, in bytes, of the extra space that must be + * present following os->end, in order to not overrun the buffer when generating + * output. When UNALIGNED_ACCESS_IS_FAST, we need at least sizeof(bitbuf_t) + * bytes for put_unaligned_leword(). Otherwise we need only 1 byte. However, + * to make the compression algorithm produce the same result on all CPU + * architectures (which is sometimes desirable), we have to unconditionally use + * the maximum for any CPU, which is sizeof(bitbuf_t) == 8. + */ +#define OUTPUT_END_PADDING 8 + +/* Initialize the output bitstream. 'size' is assumed to be at least + * OUTPUT_END_PADDING. */ +static void +deflate_init_output(struct deflate_output_bitstream *os, + void *buffer, size_t size) +{ + os->bitbuf = 0; + os->bitcount = 0; + os->begin = buffer; + os->next = os->begin; + os->end = os->begin + size - OUTPUT_END_PADDING; +} + +/* Add some bits to the bitbuffer variable of the output bitstream. The caller + * must make sure there is enough room. */ +static forceinline void +deflate_add_bits(struct deflate_output_bitstream *os, + const bitbuf_t bits, const unsigned num_bits) +{ + os->bitbuf |= bits << os->bitcount; + os->bitcount += num_bits; +} + +/* Flush bits from the bitbuffer variable to the output buffer. */ +static forceinline void +deflate_flush_bits(struct deflate_output_bitstream *os) +{ + if (UNALIGNED_ACCESS_IS_FAST) { + /* Flush a whole word (branchlessly). */ + put_unaligned_leword(os->bitbuf, os->next); + os->bitbuf >>= os->bitcount & ~7; + os->next += MIN(os->end - os->next, os->bitcount >> 3); + os->bitcount &= 7; + } else { + /* Flush a byte at a time. */ + while (os->bitcount >= 8) { + *os->next = os->bitbuf; + if (os->next != os->end) + os->next++; + os->bitcount -= 8; + os->bitbuf >>= 8; + } + } +} + +/* Align the bitstream on a byte boundary. */ +static forceinline void +deflate_align_bitstream(struct deflate_output_bitstream *os) +{ + os->bitcount += -os->bitcount & 7; + deflate_flush_bits(os); +} + +/* + * Flush any remaining bits to the output buffer if needed. Return the total + * number of bytes written to the output buffer, or 0 if an overflow occurred. + */ +static size_t +deflate_flush_output(struct deflate_output_bitstream *os) +{ + if (os->next == os->end) /* overflow? */ + return 0; + + while ((int)os->bitcount > 0) { + *os->next++ = os->bitbuf; + os->bitcount -= 8; + os->bitbuf >>= 8; + } + + return os->next - os->begin; +} + +/* Given the binary tree node A[subtree_idx] whose children already + * satisfy the maxheap property, swap the node with its greater child + * until it is greater than both its children, so that the maxheap + * property is satisfied in the subtree rooted at A[subtree_idx]. */ +static void +heapify_subtree(u32 A[], unsigned length, unsigned subtree_idx) +{ + unsigned parent_idx; + unsigned child_idx; + u32 v; + + v = A[subtree_idx]; + parent_idx = subtree_idx; + while ((child_idx = parent_idx * 2) <= length) { + if (child_idx < length && A[child_idx + 1] > A[child_idx]) + child_idx++; + if (v >= A[child_idx]) + break; + A[parent_idx] = A[child_idx]; + parent_idx = child_idx; + } + A[parent_idx] = v; +} + +/* Rearrange the array 'A' so that it satisfies the maxheap property. + * 'A' uses 1-based indices, so the children of A[i] are A[i*2] and A[i*2 + 1]. + */ +static void +heapify_array(u32 A[], unsigned length) +{ + unsigned subtree_idx; + + for (subtree_idx = length / 2; subtree_idx >= 1; subtree_idx--) + heapify_subtree(A, length, subtree_idx); +} + +/* + * Sort the array 'A', which contains 'length' unsigned 32-bit integers. + * + * Note: name this function heap_sort() instead of heapsort() to avoid colliding + * with heapsort() from stdlib.h on BSD-derived systems --- though this isn't + * necessary when compiling with -D_ANSI_SOURCE, which is the better solution. + */ +static void +heap_sort(u32 A[], unsigned length) +{ + A--; /* Use 1-based indices */ + + heapify_array(A, length); + + while (length >= 2) { + u32 tmp = A[length]; + A[length] = A[1]; + A[1] = tmp; + length--; + heapify_subtree(A, length, 1); + } +} + +#define NUM_SYMBOL_BITS 10 +#define SYMBOL_MASK ((1 << NUM_SYMBOL_BITS) - 1) + +#define GET_NUM_COUNTERS(num_syms) ((((num_syms) + 3 / 4) + 3) & ~3) +/* + * Sort the symbols primarily by frequency and secondarily by symbol + * value. Discard symbols with zero frequency and fill in an array with + * the remaining symbols, along with their frequencies. The low + * NUM_SYMBOL_BITS bits of each array entry will contain the symbol + * value, and the remaining bits will contain the frequency. + * + * @num_syms + * Number of symbols in the alphabet. + * Can't be greater than (1 << NUM_SYMBOL_BITS). + * + * @freqs[num_syms] + * The frequency of each symbol. + * + * @lens[num_syms] + * An array that eventually will hold the length of each codeword. + * This function only fills in the codeword lengths for symbols that + * have zero frequency, which are not well defined per se but will + * be set to 0. + * + * @symout[num_syms] + * The output array, described above. + * + * Returns the number of entries in 'symout' that were filled. This is + * the number of symbols that have nonzero frequency. + */ +static unsigned +sort_symbols(unsigned num_syms, const u32 freqs[restrict], + u8 lens[restrict], u32 symout[restrict]) +{ + unsigned sym; + unsigned i; + unsigned num_used_syms; + unsigned num_counters; + unsigned counters[GET_NUM_COUNTERS(DEFLATE_MAX_NUM_SYMS)]; + + /* We rely on heapsort, but with an added optimization. Since + * it's common for most symbol frequencies to be low, we first do + * a count sort using a limited number of counters. High + * frequencies will be counted in the last counter, and only they + * will be sorted with heapsort. + * + * Note: with more symbols, it is generally beneficial to have more + * counters. About 1 counter per 4 symbols seems fast. + * + * Note: I also tested radix sort, but even for large symbol + * counts (> 255) and frequencies bounded at 16 bits (enabling + * radix sort by just two base-256 digits), it didn't seem any + * faster than the method implemented here. + * + * Note: I tested the optimized quicksort implementation from + * glibc (with indirection overhead removed), but it was only + * marginally faster than the simple heapsort implemented here. + * + * Tests were done with building the codes for LZX. Results may + * vary for different compression algorithms...! */ + + num_counters = GET_NUM_COUNTERS(num_syms); + + memset(counters, 0, num_counters * sizeof(counters[0])); + + /* Count the frequencies. */ + for (sym = 0; sym < num_syms; sym++) + counters[MIN(freqs[sym], num_counters - 1)]++; + + /* Make the counters cumulative, ignoring the zero-th, which + * counted symbols with zero frequency. As a side effect, this + * calculates the number of symbols with nonzero frequency. */ + num_used_syms = 0; + for (i = 1; i < num_counters; i++) { + unsigned count = counters[i]; + counters[i] = num_used_syms; + num_used_syms += count; + } + + /* Sort nonzero-frequency symbols using the counters. At the + * same time, set the codeword lengths of zero-frequency symbols + * to 0. */ + for (sym = 0; sym < num_syms; sym++) { + u32 freq = freqs[sym]; + if (freq != 0) { + symout[counters[MIN(freq, num_counters - 1)]++] = + sym | (freq << NUM_SYMBOL_BITS); + } else { + lens[sym] = 0; + } + } + + /* Sort the symbols counted in the last counter. */ + heap_sort(symout + counters[num_counters - 2], + counters[num_counters - 1] - counters[num_counters - 2]); + + return num_used_syms; +} + +/* + * Build the Huffman tree. + * + * This is an optimized implementation that + * (a) takes advantage of the frequencies being already sorted; + * (b) only generates non-leaf nodes, since the non-leaf nodes of a + * Huffman tree are sufficient to generate a canonical code; + * (c) Only stores parent pointers, not child pointers; + * (d) Produces the nodes in the same memory used for input + * frequency information. + * + * Array 'A', which contains 'sym_count' entries, is used for both input + * and output. For this function, 'sym_count' must be at least 2. + * + * For input, the array must contain the frequencies of the symbols, + * sorted in increasing order. Specifically, each entry must contain a + * frequency left shifted by NUM_SYMBOL_BITS bits. Any data in the low + * NUM_SYMBOL_BITS bits of the entries will be ignored by this function. + * Although these bits will, in fact, contain the symbols that correspond + * to the frequencies, this function is concerned with frequencies only + * and keeps the symbols as-is. + * + * For output, this function will produce the non-leaf nodes of the + * Huffman tree. These nodes will be stored in the first (sym_count - 1) + * entries of the array. Entry A[sym_count - 2] will represent the root + * node. Each other node will contain the zero-based index of its parent + * node in 'A', left shifted by NUM_SYMBOL_BITS bits. The low + * NUM_SYMBOL_BITS bits of each entry in A will be kept as-is. Again, + * note that although these low bits will, in fact, contain a symbol + * value, this symbol will have *no relationship* with the Huffman tree + * node that happens to occupy the same slot. This is because this + * implementation only generates the non-leaf nodes of the tree. + */ +static void +build_tree(u32 A[], unsigned sym_count) +{ + /* Index, in 'A', of next lowest frequency symbol that has not + * yet been processed. */ + unsigned i = 0; + + /* Index, in 'A', of next lowest frequency parentless non-leaf + * node; or, if equal to 'e', then no such node exists yet. */ + unsigned b = 0; + + /* Index, in 'A', of next node to allocate as a non-leaf. */ + unsigned e = 0; + + do { + unsigned m, n; + u32 freq_shifted; + + /* Choose the two next lowest frequency entries. */ + + if (i != sym_count && + (b == e || (A[i] >> NUM_SYMBOL_BITS) <= (A[b] >> NUM_SYMBOL_BITS))) + m = i++; + else + m = b++; + + if (i != sym_count && + (b == e || (A[i] >> NUM_SYMBOL_BITS) <= (A[b] >> NUM_SYMBOL_BITS))) + n = i++; + else + n = b++; + + /* Allocate a non-leaf node and link the entries to it. + * + * If we link an entry that we're visiting for the first + * time (via index 'i'), then we're actually linking a + * leaf node and it will have no effect, since the leaf + * will be overwritten with a non-leaf when index 'e' + * catches up to it. But it's not any slower to + * unconditionally set the parent index. + * + * We also compute the frequency of the non-leaf node as + * the sum of its two children's frequencies. */ + + freq_shifted = (A[m] & ~SYMBOL_MASK) + (A[n] & ~SYMBOL_MASK); + + A[m] = (A[m] & SYMBOL_MASK) | (e << NUM_SYMBOL_BITS); + A[n] = (A[n] & SYMBOL_MASK) | (e << NUM_SYMBOL_BITS); + A[e] = (A[e] & SYMBOL_MASK) | freq_shifted; + e++; + } while (sym_count - e > 1); + /* When just one entry remains, it is a "leaf" that was + * linked to some other node. We ignore it, since the + * rest of the array contains the non-leaves which we + * need. (Note that we're assuming the cases with 0 or 1 + * symbols were handled separately.) */ +} + +/* + * Given the stripped-down Huffman tree constructed by build_tree(), + * determine the number of codewords that should be assigned each + * possible length, taking into account the length-limited constraint. + * + * @A + * The array produced by build_tree(), containing parent index + * information for the non-leaf nodes of the Huffman tree. Each + * entry in this array is a node; a node's parent always has a + * greater index than that node itself. This function will + * overwrite the parent index information in this array, so + * essentially it will destroy the tree. However, the data in the + * low NUM_SYMBOL_BITS of each entry will be preserved. + * + * @root_idx + * The 0-based index of the root node in 'A', and consequently one + * less than the number of tree node entries in 'A'. (Or, really 2 + * less than the actual length of 'A'.) + * + * @len_counts + * An array of length ('max_codeword_len' + 1) in which the number of + * codewords having each length <= max_codeword_len will be + * returned. + * + * @max_codeword_len + * The maximum permissible codeword length. + */ +static void +compute_length_counts(u32 A[restrict], unsigned root_idx, + unsigned len_counts[restrict], unsigned max_codeword_len) +{ + unsigned len; + int node; + + /* The key observations are: + * + * (1) We can traverse the non-leaf nodes of the tree, always + * visiting a parent before its children, by simply iterating + * through the array in reverse order. Consequently, we can + * compute the depth of each node in one pass, overwriting the + * parent indices with depths. + * + * (2) We can initially assume that in the real Huffman tree, + * both children of the root are leaves. This corresponds to two + * codewords of length 1. Then, whenever we visit a (non-leaf) + * node during the traversal, we modify this assumption to + * account for the current node *not* being a leaf, but rather + * its two children being leaves. This causes the loss of one + * codeword for the current depth and the addition of two + * codewords for the current depth plus one. + * + * (3) We can handle the length-limited constraint fairly easily + * by simply using the largest length available when a depth + * exceeds max_codeword_len. + */ + + for (len = 0; len <= max_codeword_len; len++) + len_counts[len] = 0; + len_counts[1] = 2; + + /* Set the root node's depth to 0. */ + A[root_idx] &= SYMBOL_MASK; + + for (node = root_idx - 1; node >= 0; node--) { + + /* Calculate the depth of this node. */ + + unsigned parent = A[node] >> NUM_SYMBOL_BITS; + unsigned parent_depth = A[parent] >> NUM_SYMBOL_BITS; + unsigned depth = parent_depth + 1; + unsigned len = depth; + + /* Set the depth of this node so that it is available + * when its children (if any) are processed. */ + + A[node] = (A[node] & SYMBOL_MASK) | (depth << NUM_SYMBOL_BITS); + + /* If needed, decrease the length to meet the + * length-limited constraint. This is not the optimal + * method for generating length-limited Huffman codes! + * But it should be good enough. */ + if (len >= max_codeword_len) { + len = max_codeword_len; + do { + len--; + } while (len_counts[len] == 0); + } + + /* Account for the fact that we have a non-leaf node at + * the current depth. */ + len_counts[len]--; + len_counts[len + 1] += 2; + } +} + +/* + * Generate the codewords for a canonical Huffman code. + * + * @A + * The output array for codewords. In addition, initially this + * array must contain the symbols, sorted primarily by frequency and + * secondarily by symbol value, in the low NUM_SYMBOL_BITS bits of + * each entry. + * + * @len + * Output array for codeword lengths. + * + * @len_counts + * An array that provides the number of codewords that will have + * each possible length <= max_codeword_len. + * + * @max_codeword_len + * Maximum length, in bits, of each codeword. + * + * @num_syms + * Number of symbols in the alphabet, including symbols with zero + * frequency. This is the length of the 'A' and 'len' arrays. + */ +static void +gen_codewords(u32 A[restrict], u8 lens[restrict], + const unsigned len_counts[restrict], + unsigned max_codeword_len, unsigned num_syms) +{ + u32 next_codewords[DEFLATE_MAX_CODEWORD_LEN + 1]; + unsigned i; + unsigned len; + unsigned sym; + + /* Given the number of codewords that will have each length, + * assign codeword lengths to symbols. We do this by assigning + * the lengths in decreasing order to the symbols sorted + * primarily by increasing frequency and secondarily by + * increasing symbol value. */ + for (i = 0, len = max_codeword_len; len >= 1; len--) { + unsigned count = len_counts[len]; + while (count--) + lens[A[i++] & SYMBOL_MASK] = len; + } + + /* Generate the codewords themselves. We initialize the + * 'next_codewords' array to provide the lexicographically first + * codeword of each length, then assign codewords in symbol + * order. This produces a canonical code. */ + next_codewords[0] = 0; + next_codewords[1] = 0; + for (len = 2; len <= max_codeword_len; len++) + next_codewords[len] = + (next_codewords[len - 1] + len_counts[len - 1]) << 1; + + for (sym = 0; sym < num_syms; sym++) + A[sym] = next_codewords[lens[sym]]++; +} + +/* + * --------------------------------------------------------------------- + * make_canonical_huffman_code() + * --------------------------------------------------------------------- + * + * Given an alphabet and the frequency of each symbol in it, construct a + * length-limited canonical Huffman code. + * + * @num_syms + * The number of symbols in the alphabet. The symbols are the + * integers in the range [0, num_syms - 1]. This parameter must be + * at least 2 and can't be greater than (1 << NUM_SYMBOL_BITS). + * + * @max_codeword_len + * The maximum permissible codeword length. + * + * @freqs + * An array of @num_syms entries, each of which specifies the + * frequency of the corresponding symbol. It is valid for some, + * none, or all of the frequencies to be 0. + * + * @lens + * An array of @num_syms entries in which this function will return + * the length, in bits, of the codeword assigned to each symbol. + * Symbols with 0 frequency will not have codewords per se, but + * their entries in this array will be set to 0. No lengths greater + * than @max_codeword_len will be assigned. + * + * @codewords + * An array of @num_syms entries in which this function will return + * the codeword for each symbol, right-justified and padded on the + * left with zeroes. Codewords for symbols with 0 frequency will be + * undefined. + * + * --------------------------------------------------------------------- + * + * This function builds a length-limited canonical Huffman code. + * + * A length-limited Huffman code contains no codewords longer than some + * specified length, and has exactly (with some algorithms) or + * approximately (with the algorithm used here) the minimum weighted path + * length from the root, given this constraint. + * + * A canonical Huffman code satisfies the properties that a longer + * codeword never lexicographically precedes a shorter codeword, and the + * lexicographic ordering of codewords of the same length is the same as + * the lexicographic ordering of the corresponding symbols. A canonical + * Huffman code, or more generally a canonical prefix code, can be + * reconstructed from only a list containing the codeword length of each + * symbol. + * + * The classic algorithm to generate a Huffman code creates a node for + * each symbol, then inserts these nodes into a min-heap keyed by symbol + * frequency. Then, repeatedly, the two lowest-frequency nodes are + * removed from the min-heap and added as the children of a new node + * having frequency equal to the sum of its two children, which is then + * inserted into the min-heap. When only a single node remains in the + * min-heap, it is the root of the Huffman tree. The codeword for each + * symbol is determined by the path needed to reach the corresponding + * node from the root. Descending to the left child appends a 0 bit, + * whereas descending to the right child appends a 1 bit. + * + * The classic algorithm is relatively easy to understand, but it is + * subject to a number of inefficiencies. In practice, it is fastest to + * first sort the symbols by frequency. (This itself can be subject to + * an optimization based on the fact that most frequencies tend to be + * low.) At the same time, we sort secondarily by symbol value, which + * aids the process of generating a canonical code. Then, during tree + * construction, no heap is necessary because both the leaf nodes and the + * unparented non-leaf nodes can be easily maintained in sorted order. + * Consequently, there can never be more than two possibilities for the + * next-lowest-frequency node. + * + * In addition, because we're generating a canonical code, we actually + * don't need the leaf nodes of the tree at all, only the non-leaf nodes. + * This is because for canonical code generation we don't need to know + * where the symbols are in the tree. Rather, we only need to know how + * many leaf nodes have each depth (codeword length). And this + * information can, in fact, be quickly generated from the tree of + * non-leaves only. + * + * Furthermore, we can build this stripped-down Huffman tree directly in + * the array in which the codewords are to be generated, provided that + * these array slots are large enough to hold a symbol and frequency + * value. + * + * Still furthermore, we don't even need to maintain explicit child + * pointers. We only need the parent pointers, and even those can be + * overwritten in-place with depth information as part of the process of + * extracting codeword lengths from the tree. So in summary, we do NOT + * need a big structure like: + * + * struct huffman_tree_node { + * unsigned int symbol; + * unsigned int frequency; + * unsigned int depth; + * struct huffman_tree_node *left_child; + * struct huffman_tree_node *right_child; + * }; + * + * + * ... which often gets used in "naive" implementations of Huffman code + * generation. + * + * Many of these optimizations are based on the implementation in 7-Zip + * (source file: C/HuffEnc.c), which has been placed in the public domain + * by Igor Pavlov. + */ +static void +make_canonical_huffman_code(unsigned num_syms, unsigned max_codeword_len, + const u32 freqs[restrict], + u8 lens[restrict], u32 codewords[restrict]) +{ + u32 *A = codewords; + unsigned num_used_syms; + + STATIC_ASSERT(DEFLATE_MAX_NUM_SYMS <= 1 << NUM_SYMBOL_BITS); + + /* We begin by sorting the symbols primarily by frequency and + * secondarily by symbol value. As an optimization, the array + * used for this purpose ('A') shares storage with the space in + * which we will eventually return the codewords. */ + + num_used_syms = sort_symbols(num_syms, freqs, lens, A); + + /* 'num_used_syms' is the number of symbols with nonzero + * frequency. This may be less than @num_syms. 'num_used_syms' + * is also the number of entries in 'A' that are valid. Each + * entry consists of a distinct symbol and a nonzero frequency + * packed into a 32-bit integer. */ + + /* Handle special cases where only 0 or 1 symbols were used (had + * nonzero frequency). */ + + if (unlikely(num_used_syms == 0)) { + /* Code is empty. sort_symbols() already set all lengths + * to 0, so there is nothing more to do. */ + return; + } + + if (unlikely(num_used_syms == 1)) { + /* Only one symbol was used, so we only need one + * codeword. But two codewords are needed to form the + * smallest complete Huffman code, which uses codewords 0 + * and 1. Therefore, we choose another symbol to which + * to assign a codeword. We use 0 (if the used symbol is + * not 0) or 1 (if the used symbol is 0). In either + * case, the lesser-valued symbol must be assigned + * codeword 0 so that the resulting code is canonical. */ + + unsigned sym = A[0] & SYMBOL_MASK; + unsigned nonzero_idx = sym ? sym : 1; + + codewords[0] = 0; + lens[0] = 1; + codewords[nonzero_idx] = 1; + lens[nonzero_idx] = 1; + return; + } + + /* Build a stripped-down version of the Huffman tree, sharing the + * array 'A' with the symbol values. Then extract length counts + * from the tree and use them to generate the final codewords. */ + + build_tree(A, num_used_syms); + + { + unsigned len_counts[DEFLATE_MAX_CODEWORD_LEN + 1]; + + compute_length_counts(A, num_used_syms - 2, + len_counts, max_codeword_len); + + gen_codewords(A, lens, len_counts, max_codeword_len, num_syms); + } +} + +/* + * Clear the Huffman symbol frequency counters. + * This must be called when starting a new DEFLATE block. + */ +static void +deflate_reset_symbol_frequencies(struct libdeflate_compressor *c) +{ + memset(&c->freqs, 0, sizeof(c->freqs)); +} + +/* Reverse the Huffman codeword 'codeword', which is 'len' bits in length. */ +static u32 +deflate_reverse_codeword(u32 codeword, u8 len) +{ + /* The following branchless algorithm is faster than going bit by bit. + * Note: since no codewords are longer than 16 bits, we only need to + * reverse the low 16 bits of the 'u32'. */ + STATIC_ASSERT(DEFLATE_MAX_CODEWORD_LEN <= 16); + + /* Flip adjacent 1-bit fields */ + codeword = ((codeword & 0x5555) << 1) | ((codeword & 0xAAAA) >> 1); + + /* Flip adjacent 2-bit fields */ + codeword = ((codeword & 0x3333) << 2) | ((codeword & 0xCCCC) >> 2); + + /* Flip adjacent 4-bit fields */ + codeword = ((codeword & 0x0F0F) << 4) | ((codeword & 0xF0F0) >> 4); + + /* Flip adjacent 8-bit fields */ + codeword = ((codeword & 0x00FF) << 8) | ((codeword & 0xFF00) >> 8); + + /* Return the high 'len' bits of the bit-reversed 16 bit value. */ + return codeword >> (16 - len); +} + +/* Make a canonical Huffman code with bit-reversed codewords. */ +static void +deflate_make_huffman_code(unsigned num_syms, unsigned max_codeword_len, + const u32 freqs[], u8 lens[], u32 codewords[]) +{ + unsigned sym; + + make_canonical_huffman_code(num_syms, max_codeword_len, + freqs, lens, codewords); + + for (sym = 0; sym < num_syms; sym++) + codewords[sym] = deflate_reverse_codeword(codewords[sym], lens[sym]); +} + +/* + * Build the literal/length and offset Huffman codes for a DEFLATE block. + * + * This takes as input the frequency tables for each code and produces as output + * a set of tables that map symbols to codewords and codeword lengths. + */ +static void +deflate_make_huffman_codes(const struct deflate_freqs *freqs, + struct deflate_codes *codes) +{ + STATIC_ASSERT(MAX_LITLEN_CODEWORD_LEN <= DEFLATE_MAX_LITLEN_CODEWORD_LEN); + STATIC_ASSERT(MAX_OFFSET_CODEWORD_LEN <= DEFLATE_MAX_OFFSET_CODEWORD_LEN); + + deflate_make_huffman_code(DEFLATE_NUM_LITLEN_SYMS, + MAX_LITLEN_CODEWORD_LEN, + freqs->litlen, + codes->lens.litlen, + codes->codewords.litlen); + + deflate_make_huffman_code(DEFLATE_NUM_OFFSET_SYMS, + MAX_OFFSET_CODEWORD_LEN, + freqs->offset, + codes->lens.offset, + codes->codewords.offset); +} + +/* Initialize c->static_codes. */ +static void +deflate_init_static_codes(struct libdeflate_compressor *c) +{ + unsigned i; + + for (i = 0; i < 144; i++) + c->freqs.litlen[i] = 1 << (9 - 8); + for (; i < 256; i++) + c->freqs.litlen[i] = 1 << (9 - 9); + for (; i < 280; i++) + c->freqs.litlen[i] = 1 << (9 - 7); + for (; i < 288; i++) + c->freqs.litlen[i] = 1 << (9 - 8); + + for (i = 0; i < 32; i++) + c->freqs.offset[i] = 1 << (5 - 5); + + deflate_make_huffman_codes(&c->freqs, &c->static_codes); +} + +/* Return the offset slot for the specified match offset. */ +static forceinline unsigned +deflate_get_offset_slot(struct libdeflate_compressor *c, unsigned offset) +{ +#if USE_FULL_OFFSET_SLOT_FAST + return c->offset_slot_fast[offset]; +#else + if (offset <= 256) + return c->offset_slot_fast[offset - 1]; + else + return c->offset_slot_fast[256 + ((offset - 1) >> 7)]; +#endif +} + +/* Write the header fields common to all DEFLATE block types. */ +static void +deflate_write_block_header(struct deflate_output_bitstream *os, + bool is_final_block, unsigned block_type) +{ + deflate_add_bits(os, is_final_block, 1); + deflate_add_bits(os, block_type, 2); + deflate_flush_bits(os); +} + +static unsigned +deflate_compute_precode_items(const u8 lens[restrict], + const unsigned num_lens, + u32 precode_freqs[restrict], + unsigned precode_items[restrict]) +{ + unsigned *itemptr; + unsigned run_start; + unsigned run_end; + unsigned extra_bits; + u8 len; + + memset(precode_freqs, 0, + DEFLATE_NUM_PRECODE_SYMS * sizeof(precode_freqs[0])); + + itemptr = precode_items; + run_start = 0; + do { + /* Find the next run of codeword lengths. */ + + /* len = the length being repeated */ + len = lens[run_start]; + + /* Extend the run. */ + run_end = run_start; + do { + run_end++; + } while (run_end != num_lens && len == lens[run_end]); + + if (len == 0) { + /* Run of zeroes. */ + + /* Symbol 18: RLE 11 to 138 zeroes at a time. */ + while ((run_end - run_start) >= 11) { + extra_bits = MIN((run_end - run_start) - 11, 0x7F); + precode_freqs[18]++; + *itemptr++ = 18 | (extra_bits << 5); + run_start += 11 + extra_bits; + } + + /* Symbol 17: RLE 3 to 10 zeroes at a time. */ + if ((run_end - run_start) >= 3) { + extra_bits = MIN((run_end - run_start) - 3, 0x7); + precode_freqs[17]++; + *itemptr++ = 17 | (extra_bits << 5); + run_start += 3 + extra_bits; + } + } else { + + /* A run of nonzero lengths. */ + + /* Symbol 16: RLE 3 to 6 of the previous length. */ + if ((run_end - run_start) >= 4) { + precode_freqs[len]++; + *itemptr++ = len; + run_start++; + do { + extra_bits = MIN((run_end - run_start) - 3, 0x3); + precode_freqs[16]++; + *itemptr++ = 16 | (extra_bits << 5); + run_start += 3 + extra_bits; + } while ((run_end - run_start) >= 3); + } + } + + /* Output any remaining lengths without RLE. */ + while (run_start != run_end) { + precode_freqs[len]++; + *itemptr++ = len; + run_start++; + } + } while (run_start != num_lens); + + return itemptr - precode_items; +} + +/* + * Huffman codeword lengths for dynamic Huffman blocks are compressed using a + * separate Huffman code, the "precode", which contains a symbol for each + * possible codeword length in the larger code as well as several special + * symbols to represent repeated codeword lengths (a form of run-length + * encoding). The precode is itself constructed in canonical form, and its + * codeword lengths are represented literally in 19 3-bit fields that + * immediately precede the compressed codeword lengths of the larger code. + */ + +/* Precompute the information needed to output Huffman codes. */ +static void +deflate_precompute_huffman_header(struct libdeflate_compressor *c) +{ + /* Compute how many litlen and offset symbols are needed. */ + + for (c->num_litlen_syms = DEFLATE_NUM_LITLEN_SYMS; + c->num_litlen_syms > 257; + c->num_litlen_syms--) + if (c->codes.lens.litlen[c->num_litlen_syms - 1] != 0) + break; + + for (c->num_offset_syms = DEFLATE_NUM_OFFSET_SYMS; + c->num_offset_syms > 1; + c->num_offset_syms--) + if (c->codes.lens.offset[c->num_offset_syms - 1] != 0) + break; + + /* If we're not using the full set of literal/length codeword lengths, + * then temporarily move the offset codeword lengths over so that the + * literal/length and offset codeword lengths are contiguous. */ + + STATIC_ASSERT(offsetof(struct deflate_lens, offset) == + DEFLATE_NUM_LITLEN_SYMS); + + if (c->num_litlen_syms != DEFLATE_NUM_LITLEN_SYMS) { + memmove((u8 *)&c->codes.lens + c->num_litlen_syms, + (u8 *)&c->codes.lens + DEFLATE_NUM_LITLEN_SYMS, + c->num_offset_syms); + } + + /* Compute the "items" (RLE / literal tokens and extra bits) with which + * the codeword lengths in the larger code will be output. */ + c->num_precode_items = + deflate_compute_precode_items((u8 *)&c->codes.lens, + c->num_litlen_syms + + c->num_offset_syms, + c->precode_freqs, + c->precode_items); + + /* Build the precode. */ + STATIC_ASSERT(MAX_PRE_CODEWORD_LEN <= DEFLATE_MAX_PRE_CODEWORD_LEN); + deflate_make_huffman_code(DEFLATE_NUM_PRECODE_SYMS, + MAX_PRE_CODEWORD_LEN, + c->precode_freqs, c->precode_lens, + c->precode_codewords); + + /* Count how many precode lengths we actually need to output. */ + for (c->num_explicit_lens = DEFLATE_NUM_PRECODE_SYMS; + c->num_explicit_lens > 4; + c->num_explicit_lens--) + if (c->precode_lens[deflate_precode_lens_permutation[ + c->num_explicit_lens - 1]] != 0) + break; + + /* Restore the offset codeword lengths if needed. */ + if (c->num_litlen_syms != DEFLATE_NUM_LITLEN_SYMS) { + memmove((u8 *)&c->codes.lens + DEFLATE_NUM_LITLEN_SYMS, + (u8 *)&c->codes.lens + c->num_litlen_syms, + c->num_offset_syms); + } +} + +/* Output the Huffman codes. */ +static void +deflate_write_huffman_header(struct libdeflate_compressor *c, + struct deflate_output_bitstream *os) +{ + unsigned i; + + deflate_add_bits(os, c->num_litlen_syms - 257, 5); + deflate_add_bits(os, c->num_offset_syms - 1, 5); + deflate_add_bits(os, c->num_explicit_lens - 4, 4); + deflate_flush_bits(os); + + /* Output the lengths of the codewords in the precode. */ + for (i = 0; i < c->num_explicit_lens; i++) { + deflate_add_bits(os, c->precode_lens[ + deflate_precode_lens_permutation[i]], 3); + deflate_flush_bits(os); + } + + /* Output the encoded lengths of the codewords in the larger code. */ + for (i = 0; i < c->num_precode_items; i++) { + unsigned precode_item = c->precode_items[i]; + unsigned precode_sym = precode_item & 0x1F; + deflate_add_bits(os, c->precode_codewords[precode_sym], + c->precode_lens[precode_sym]); + if (precode_sym >= 16) { + if (precode_sym == 16) + deflate_add_bits(os, precode_item >> 5, 2); + else if (precode_sym == 17) + deflate_add_bits(os, precode_item >> 5, 3); + else + deflate_add_bits(os, precode_item >> 5, 7); + } + STATIC_ASSERT(CAN_BUFFER(DEFLATE_MAX_PRE_CODEWORD_LEN + 7)); + deflate_flush_bits(os); + } +} + +static void +deflate_write_sequences(struct deflate_output_bitstream * restrict os, + const struct deflate_codes * restrict codes, + const struct deflate_sequence sequences[restrict], + const u8 * restrict in_next) +{ + const struct deflate_sequence *seq = sequences; + + for (;;) { + u32 litrunlen = seq->litrunlen_and_length & 0x7FFFFF; + unsigned length = seq->litrunlen_and_length >> 23; + unsigned length_slot; + unsigned litlen_symbol; + unsigned offset_symbol; + + if (litrunlen) { + #if 1 + while (litrunlen >= 4) { + unsigned lit0 = in_next[0]; + unsigned lit1 = in_next[1]; + unsigned lit2 = in_next[2]; + unsigned lit3 = in_next[3]; + + deflate_add_bits(os, codes->codewords.litlen[lit0], + codes->lens.litlen[lit0]); + if (!CAN_BUFFER(2 * MAX_LITLEN_CODEWORD_LEN)) + deflate_flush_bits(os); + + deflate_add_bits(os, codes->codewords.litlen[lit1], + codes->lens.litlen[lit1]); + if (!CAN_BUFFER(4 * MAX_LITLEN_CODEWORD_LEN)) + deflate_flush_bits(os); + + deflate_add_bits(os, codes->codewords.litlen[lit2], + codes->lens.litlen[lit2]); + if (!CAN_BUFFER(2 * MAX_LITLEN_CODEWORD_LEN)) + deflate_flush_bits(os); + + deflate_add_bits(os, codes->codewords.litlen[lit3], + codes->lens.litlen[lit3]); + deflate_flush_bits(os); + in_next += 4; + litrunlen -= 4; + } + if (litrunlen-- != 0) { + deflate_add_bits(os, codes->codewords.litlen[*in_next], + codes->lens.litlen[*in_next]); + if (!CAN_BUFFER(3 * MAX_LITLEN_CODEWORD_LEN)) + deflate_flush_bits(os); + in_next++; + if (litrunlen-- != 0) { + deflate_add_bits(os, codes->codewords.litlen[*in_next], + codes->lens.litlen[*in_next]); + if (!CAN_BUFFER(3 * MAX_LITLEN_CODEWORD_LEN)) + deflate_flush_bits(os); + in_next++; + if (litrunlen-- != 0) { + deflate_add_bits(os, codes->codewords.litlen[*in_next], + codes->lens.litlen[*in_next]); + if (!CAN_BUFFER(3 * MAX_LITLEN_CODEWORD_LEN)) + deflate_flush_bits(os); + in_next++; + } + } + if (CAN_BUFFER(3 * MAX_LITLEN_CODEWORD_LEN)) + deflate_flush_bits(os); + } + #else + do { + unsigned lit = *in_next++; + deflate_add_bits(os, codes->codewords.litlen[lit], + codes->lens.litlen[lit]); + deflate_flush_bits(os); + } while (--litrunlen); + #endif + } + + if (length == 0) + return; + + in_next += length; + + length_slot = seq->length_slot; + litlen_symbol = 257 + length_slot; + + /* Litlen symbol */ + deflate_add_bits(os, codes->codewords.litlen[litlen_symbol], + codes->lens.litlen[litlen_symbol]); + + /* Extra length bits */ + STATIC_ASSERT(CAN_BUFFER(MAX_LITLEN_CODEWORD_LEN + + DEFLATE_MAX_EXTRA_LENGTH_BITS)); + deflate_add_bits(os, length - deflate_length_slot_base[length_slot], + deflate_extra_length_bits[length_slot]); + + if (!CAN_BUFFER(MAX_LITLEN_CODEWORD_LEN + + DEFLATE_MAX_EXTRA_LENGTH_BITS + + MAX_OFFSET_CODEWORD_LEN + + DEFLATE_MAX_EXTRA_OFFSET_BITS)) + deflate_flush_bits(os); + + /* Offset symbol */ + offset_symbol = seq->offset_symbol; + deflate_add_bits(os, codes->codewords.offset[offset_symbol], + codes->lens.offset[offset_symbol]); + + if (!CAN_BUFFER(MAX_OFFSET_CODEWORD_LEN + + DEFLATE_MAX_EXTRA_OFFSET_BITS)) + deflate_flush_bits(os); + + /* Extra offset bits */ + deflate_add_bits(os, seq->offset - deflate_offset_slot_base[offset_symbol], + deflate_extra_offset_bits[offset_symbol]); + + deflate_flush_bits(os); + + seq++; + } +} + +#if SUPPORT_NEAR_OPTIMAL_PARSING +/* + * Follow the minimum-cost path in the graph of possible match/literal choices + * for the current block and write out the matches/literals using the specified + * Huffman codes. + * + * Note: this is slightly duplicated with deflate_write_sequences(), the reason + * being that we don't want to waste time translating between intermediate + * match/literal representations. + */ +static void +deflate_write_item_list(struct deflate_output_bitstream *os, + const struct deflate_codes *codes, + struct libdeflate_compressor *c, + u32 block_length) +{ + struct deflate_optimum_node *cur_node = &c->p.n.optimum_nodes[0]; + struct deflate_optimum_node * const end_node = &c->p.n.optimum_nodes[block_length]; + do { + unsigned length = cur_node->item & OPTIMUM_LEN_MASK; + unsigned offset = cur_node->item >> OPTIMUM_OFFSET_SHIFT; + unsigned litlen_symbol; + unsigned length_slot; + unsigned offset_slot; + + if (length == 1) { + /* Literal */ + litlen_symbol = offset; + deflate_add_bits(os, codes->codewords.litlen[litlen_symbol], + codes->lens.litlen[litlen_symbol]); + deflate_flush_bits(os); + } else { + /* Match length */ + length_slot = deflate_length_slot[length]; + litlen_symbol = 257 + length_slot; + deflate_add_bits(os, codes->codewords.litlen[litlen_symbol], + codes->lens.litlen[litlen_symbol]); + + deflate_add_bits(os, length - deflate_length_slot_base[length_slot], + deflate_extra_length_bits[length_slot]); + + if (!CAN_BUFFER(MAX_LITLEN_CODEWORD_LEN + + DEFLATE_MAX_EXTRA_LENGTH_BITS + + MAX_OFFSET_CODEWORD_LEN + + DEFLATE_MAX_EXTRA_OFFSET_BITS)) + deflate_flush_bits(os); + + + /* Match offset */ + offset_slot = deflate_get_offset_slot(c, offset); + deflate_add_bits(os, codes->codewords.offset[offset_slot], + codes->lens.offset[offset_slot]); + + if (!CAN_BUFFER(MAX_OFFSET_CODEWORD_LEN + + DEFLATE_MAX_EXTRA_OFFSET_BITS)) + deflate_flush_bits(os); + + deflate_add_bits(os, offset - deflate_offset_slot_base[offset_slot], + deflate_extra_offset_bits[offset_slot]); + + deflate_flush_bits(os); + } + cur_node += length; + } while (cur_node != end_node); +} +#endif /* SUPPORT_NEAR_OPTIMAL_PARSING */ + +/* Output the end-of-block symbol. */ +static void +deflate_write_end_of_block(struct deflate_output_bitstream *os, + const struct deflate_codes *codes) +{ + deflate_add_bits(os, codes->codewords.litlen[DEFLATE_END_OF_BLOCK], + codes->lens.litlen[DEFLATE_END_OF_BLOCK]); + deflate_flush_bits(os); +} + +static void +deflate_write_uncompressed_block(struct deflate_output_bitstream *os, + const u8 *data, u16 len, + bool is_final_block) +{ + deflate_write_block_header(os, is_final_block, + DEFLATE_BLOCKTYPE_UNCOMPRESSED); + deflate_align_bitstream(os); + + if (4 + (u32)len >= os->end - os->next) { + os->next = os->end; + return; + } + + put_unaligned_le16(len, os->next); + os->next += 2; + put_unaligned_le16(~len, os->next); + os->next += 2; + memcpy(os->next, data, len); + os->next += len; +} + +static void +deflate_write_uncompressed_blocks(struct deflate_output_bitstream *os, + const u8 *data, size_t data_length, + bool is_final_block) +{ + do { + u16 len = MIN(data_length, UINT16_MAX); + + deflate_write_uncompressed_block(os, data, len, + is_final_block && len == data_length); + data += len; + data_length -= len; + } while (data_length != 0); +} + +/* + * Choose the best type of block to use (dynamic Huffman, static Huffman, or + * uncompressed), then output it. + */ +static void +deflate_flush_block(struct libdeflate_compressor * restrict c, + struct deflate_output_bitstream * restrict os, + const u8 * restrict block_begin, u32 block_length, + bool is_final_block, bool use_item_list) +{ + static const u8 deflate_extra_precode_bits[DEFLATE_NUM_PRECODE_SYMS] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 7, + }; + + /* Costs are measured in bits */ + u32 dynamic_cost = 0; + u32 static_cost = 0; + u32 uncompressed_cost = 0; + struct deflate_codes *codes; + int block_type; + unsigned sym; + + /* Tally the end-of-block symbol. */ + c->freqs.litlen[DEFLATE_END_OF_BLOCK]++; + + /* Build dynamic Huffman codes. */ + deflate_make_huffman_codes(&c->freqs, &c->codes); + + /* Account for the cost of sending dynamic Huffman codes. */ + deflate_precompute_huffman_header(c); + dynamic_cost += 5 + 5 + 4 + (3 * c->num_explicit_lens); + for (sym = 0; sym < DEFLATE_NUM_PRECODE_SYMS; sym++) { + u32 extra = deflate_extra_precode_bits[sym]; + dynamic_cost += c->precode_freqs[sym] * + (extra + c->precode_lens[sym]); + } + + /* Account for the cost of encoding literals. */ + for (sym = 0; sym < 256; sym++) { + dynamic_cost += c->freqs.litlen[sym] * + c->codes.lens.litlen[sym]; + } + for (sym = 0; sym < 144; sym++) + static_cost += c->freqs.litlen[sym] * 8; + for (; sym < 256; sym++) + static_cost += c->freqs.litlen[sym] * 9; + + /* Account for the cost of encoding the end-of-block symbol. */ + dynamic_cost += c->codes.lens.litlen[256]; + static_cost += 7; + + /* Account for the cost of encoding lengths. */ + for (sym = 257; sym < 257 + ARRAY_LEN(deflate_extra_length_bits); sym++) { + u32 extra = deflate_extra_length_bits[sym - 257]; + dynamic_cost += c->freqs.litlen[sym] * + (extra + c->codes.lens.litlen[sym]); + static_cost += c->freqs.litlen[sym] * + (extra + c->static_codes.lens.litlen[sym]); + } + + /* Account for the cost of encoding offsets. */ + for (sym = 0; sym < ARRAY_LEN(deflate_extra_offset_bits); sym++) { + u32 extra = deflate_extra_offset_bits[sym]; + dynamic_cost += c->freqs.offset[sym] * + (extra + c->codes.lens.offset[sym]); + static_cost += c->freqs.offset[sym] * (extra + 5); + } + + /* Compute the cost of using uncompressed blocks. */ + uncompressed_cost += (-(os->bitcount + 3) & 7) + 32 + + (40 * (DIV_ROUND_UP(block_length, + UINT16_MAX) - 1)) + + (8 * block_length); + + /* Choose the cheapest block type. */ + if (dynamic_cost < MIN(static_cost, uncompressed_cost)) { + block_type = DEFLATE_BLOCKTYPE_DYNAMIC_HUFFMAN; + codes = &c->codes; + } else if (static_cost < uncompressed_cost) { + block_type = DEFLATE_BLOCKTYPE_STATIC_HUFFMAN; + codes = &c->static_codes; + } else { + block_type = DEFLATE_BLOCKTYPE_UNCOMPRESSED; + } + + /* Now actually output the block. */ + + if (block_type == DEFLATE_BLOCKTYPE_UNCOMPRESSED) { + /* Note: the length being flushed may exceed the maximum length + * of an uncompressed block (65535 bytes). Therefore, more than + * one uncompressed block might be needed. */ + deflate_write_uncompressed_blocks(os, block_begin, block_length, + is_final_block); + } else { + /* Output the block header. */ + deflate_write_block_header(os, is_final_block, block_type); + + /* Output the Huffman codes (dynamic Huffman blocks only). */ + if (block_type == DEFLATE_BLOCKTYPE_DYNAMIC_HUFFMAN) + deflate_write_huffman_header(c, os); + + /* Output the literals, matches, and end-of-block symbol. */ + #if SUPPORT_NEAR_OPTIMAL_PARSING + if (use_item_list) + deflate_write_item_list(os, codes, c, block_length); + else + #endif + deflate_write_sequences(os, codes, c->p.g.sequences, + block_begin); + deflate_write_end_of_block(os, codes); + } +} + +static forceinline void +deflate_choose_literal(struct libdeflate_compressor *c, unsigned literal, + u32 *litrunlen_p) +{ + c->freqs.litlen[literal]++; + ++*litrunlen_p; +} + +static forceinline void +deflate_choose_match(struct libdeflate_compressor *c, + unsigned length, unsigned offset, + u32 *litrunlen_p, struct deflate_sequence **next_seq_p) +{ + struct deflate_sequence *seq = *next_seq_p; + unsigned length_slot = deflate_length_slot[length]; + unsigned offset_slot = deflate_get_offset_slot(c, offset); + + c->freqs.litlen[257 + length_slot]++; + c->freqs.offset[offset_slot]++; + + seq->litrunlen_and_length = ((u32)length << 23) | *litrunlen_p; + seq->offset = offset; + seq->length_slot = length_slot; + seq->offset_symbol = offset_slot; + + *litrunlen_p = 0; + *next_seq_p = seq + 1; +} + +static forceinline void +deflate_finish_sequence(struct deflate_sequence *seq, u32 litrunlen) +{ + seq->litrunlen_and_length = litrunlen; /* length = 0 */ +} + +/******************************************************************************/ + +/* + * Block splitting algorithm. The problem is to decide when it is worthwhile to + * start a new block with new Huffman codes. There is a theoretically optimal + * solution: recursively consider every possible block split, considering the + * exact cost of each block, and choose the minimum cost approach. But this is + * far too slow. Instead, as an approximation, we can count symbols and after + * every N symbols, compare the expected distribution of symbols based on the + * previous data with the actual distribution. If they differ "by enough", then + * start a new block. + * + * As an optimization and heuristic, we don't distinguish between every symbol + * but rather we combine many symbols into a single "observation type". For + * literals we only look at the high bits and low bits, and for matches we only + * look at whether the match is long or not. The assumption is that for typical + * "real" data, places that are good block boundaries will tend to be noticeable + * based only on changes in these aggregate frequencies, without looking for + * subtle differences in individual symbols. For example, a change from ASCII + * bytes to non-ASCII bytes, or from few matches (generally less compressible) + * to many matches (generally more compressible), would be easily noticed based + * on the aggregates. + * + * For determining whether the frequency distributions are "different enough" to + * start a new block, the simply heuristic of splitting when the sum of absolute + * differences exceeds a constant seems to be good enough. We also add a number + * proportional to the block length so that the algorithm is more likely to end + * long blocks than short blocks. This reflects the general expectation that it + * will become increasingly beneficial to start a new block as the current + * block grows longer. + * + * Finally, for an approximation, it is not strictly necessary that the exact + * symbols being used are considered. With "near-optimal parsing", for example, + * the actual symbols that will be used are unknown until after the block + * boundary is chosen and the block has been optimized. Since the final choices + * cannot be used, we can use preliminary "greedy" choices instead. + */ + +/* Initialize the block split statistics when starting a new block. */ +static void +init_block_split_stats(struct block_split_stats *stats) +{ + int i; + + for (i = 0; i < NUM_OBSERVATION_TYPES; i++) { + stats->new_observations[i] = 0; + stats->observations[i] = 0; + } + stats->num_new_observations = 0; + stats->num_observations = 0; +} + +/* Literal observation. Heuristic: use the top 2 bits and low 1 bits of the + * literal, for 8 possible literal observation types. */ +static forceinline void +observe_literal(struct block_split_stats *stats, u8 lit) +{ + stats->new_observations[((lit >> 5) & 0x6) | (lit & 1)]++; + stats->num_new_observations++; +} + +/* Match observation. Heuristic: use one observation type for "short match" and + * one observation type for "long match". */ +static forceinline void +observe_match(struct block_split_stats *stats, unsigned length) +{ + stats->new_observations[NUM_LITERAL_OBSERVATION_TYPES + (length >= 9)]++; + stats->num_new_observations++; +} + +static bool +do_end_block_check(struct block_split_stats *stats, u32 block_length) +{ + int i; + + if (stats->num_observations > 0) { + + /* Note: to avoid slow divisions, we do not divide by + * 'num_observations', but rather do all math with the numbers + * multiplied by 'num_observations'. */ + u32 total_delta = 0; + for (i = 0; i < NUM_OBSERVATION_TYPES; i++) { + u32 expected = stats->observations[i] * stats->num_new_observations; + u32 actual = stats->new_observations[i] * stats->num_observations; + u32 delta = (actual > expected) ? actual - expected : + expected - actual; + total_delta += delta; + } + + /* Ready to end the block? */ + if (total_delta + (block_length / 4096) * stats->num_observations >= + NUM_OBSERVATIONS_PER_BLOCK_CHECK * 200 / 512 * stats->num_observations) + return true; + } + + for (i = 0; i < NUM_OBSERVATION_TYPES; i++) { + stats->num_observations += stats->new_observations[i]; + stats->observations[i] += stats->new_observations[i]; + stats->new_observations[i] = 0; + } + stats->num_new_observations = 0; + return false; +} + +static forceinline bool +should_end_block(struct block_split_stats *stats, + const u8 *in_block_begin, const u8 *in_next, const u8 *in_end) +{ + /* Ready to check block split statistics? */ + if (stats->num_new_observations < NUM_OBSERVATIONS_PER_BLOCK_CHECK || + in_next - in_block_begin < MIN_BLOCK_LENGTH || + in_end - in_next < MIN_BLOCK_LENGTH) + return false; + + return do_end_block_check(stats, in_next - in_block_begin); +} + +/******************************************************************************/ + +/* + * This is the level 0 "compressor". It always outputs uncompressed blocks. + */ +static size_t +deflate_compress_none(struct libdeflate_compressor * restrict c, + const u8 * restrict in, size_t in_nbytes, + u8 * restrict out, size_t out_nbytes_avail) +{ + struct deflate_output_bitstream os; + + deflate_init_output(&os, out, out_nbytes_avail); + + deflate_write_uncompressed_blocks(&os, in, in_nbytes, true); + + return deflate_flush_output(&os); +} + +/* + * This is the "greedy" DEFLATE compressor. It always chooses the longest match. + */ +static size_t +deflate_compress_greedy(struct libdeflate_compressor * restrict c, + const u8 * restrict in, size_t in_nbytes, + u8 * restrict out, size_t out_nbytes_avail) +{ + const u8 *in_next = in; + const u8 *in_end = in_next + in_nbytes; + struct deflate_output_bitstream os; + const u8 *in_cur_base = in_next; + unsigned max_len = DEFLATE_MAX_MATCH_LEN; + unsigned nice_len = MIN(c->nice_match_length, max_len); + u32 next_hashes[2] = {0, 0}; + + deflate_init_output(&os, out, out_nbytes_avail); + hc_matchfinder_init(&c->p.g.hc_mf); + + do { + /* Starting a new DEFLATE block. */ + + const u8 * const in_block_begin = in_next; + const u8 * const in_max_block_end = + in_next + MIN(in_end - in_next, SOFT_MAX_BLOCK_LENGTH); + u32 litrunlen = 0; + struct deflate_sequence *next_seq = c->p.g.sequences; + + init_block_split_stats(&c->split_stats); + deflate_reset_symbol_frequencies(c); + + do { + u32 length; + u32 offset; + + /* Decrease the maximum and nice match lengths if we're + * approaching the end of the input buffer. */ + if (unlikely(max_len > in_end - in_next)) { + max_len = in_end - in_next; + nice_len = MIN(nice_len, max_len); + } + + length = hc_matchfinder_longest_match(&c->p.g.hc_mf, + &in_cur_base, + in_next, + DEFLATE_MIN_MATCH_LEN - 1, + max_len, + nice_len, + c->max_search_depth, + next_hashes, + &offset); + + if (length >= DEFLATE_MIN_MATCH_LEN) { + /* Match found. */ + deflate_choose_match(c, length, offset, + &litrunlen, &next_seq); + observe_match(&c->split_stats, length); + in_next = hc_matchfinder_skip_positions(&c->p.g.hc_mf, + &in_cur_base, + in_next + 1, + in_end, + length - 1, + next_hashes); + } else { + /* No match found. */ + deflate_choose_literal(c, *in_next, &litrunlen); + observe_literal(&c->split_stats, *in_next); + in_next++; + } + + /* Check if it's time to output another block. */ + } while (in_next < in_max_block_end && + !should_end_block(&c->split_stats, in_block_begin, in_next, in_end)); + + deflate_finish_sequence(next_seq, litrunlen); + deflate_flush_block(c, &os, in_block_begin, + in_next - in_block_begin, + in_next == in_end, false); + } while (in_next != in_end); + + return deflate_flush_output(&os); +} + +/* + * This is the "lazy" DEFLATE compressor. Before choosing a match, it checks to + * see if there's a longer match at the next position. If yes, it outputs a + * literal and continues to the next position. If no, it outputs the match. + */ +static size_t +deflate_compress_lazy(struct libdeflate_compressor * restrict c, + const u8 * restrict in, size_t in_nbytes, + u8 * restrict out, size_t out_nbytes_avail) +{ + const u8 *in_next = in; + const u8 *in_end = in_next + in_nbytes; + struct deflate_output_bitstream os; + const u8 *in_cur_base = in_next; + unsigned max_len = DEFLATE_MAX_MATCH_LEN; + unsigned nice_len = MIN(c->nice_match_length, max_len); + u32 next_hashes[2] = {0, 0}; + + deflate_init_output(&os, out, out_nbytes_avail); + hc_matchfinder_init(&c->p.g.hc_mf); + + do { + /* Starting a new DEFLATE block. */ + + const u8 * const in_block_begin = in_next; + const u8 * const in_max_block_end = + in_next + MIN(in_end - in_next, SOFT_MAX_BLOCK_LENGTH); + u32 litrunlen = 0; + struct deflate_sequence *next_seq = c->p.g.sequences; + + init_block_split_stats(&c->split_stats); + deflate_reset_symbol_frequencies(c); + + do { + unsigned cur_len; + unsigned cur_offset; + unsigned next_len; + unsigned next_offset; + + if (unlikely(in_end - in_next < DEFLATE_MAX_MATCH_LEN)) { + max_len = in_end - in_next; + nice_len = MIN(nice_len, max_len); + } + + /* Find the longest match at the current position. */ + cur_len = hc_matchfinder_longest_match(&c->p.g.hc_mf, + &in_cur_base, + in_next, + DEFLATE_MIN_MATCH_LEN - 1, + max_len, + nice_len, + c->max_search_depth, + next_hashes, + &cur_offset); + in_next += 1; + + if (cur_len < DEFLATE_MIN_MATCH_LEN) { + /* No match found. Choose a literal. */ + deflate_choose_literal(c, *(in_next - 1), &litrunlen); + observe_literal(&c->split_stats, *(in_next - 1)); + continue; + } + + have_cur_match: + observe_match(&c->split_stats, cur_len); + + /* We have a match at the current position. */ + + /* If the current match is very long, choose it + * immediately. */ + if (cur_len >= nice_len) { + deflate_choose_match(c, cur_len, cur_offset, + &litrunlen, &next_seq); + in_next = hc_matchfinder_skip_positions(&c->p.g.hc_mf, + &in_cur_base, + in_next, + in_end, + cur_len - 1, + next_hashes); + continue; + } + + /* + * Try to find a match at the next position. + * + * Note: since we already have a match at the *current* + * position, we use only half the 'max_search_depth' + * when checking the *next* position. This is a useful + * trade-off because it's more worthwhile to use a + * greater search depth on the initial match. + * + * Note: it's possible to structure the code such that + * there's only one call to longest_match(), which + * handles both the "find the initial match" and "try to + * find a longer match" cases. However, it is faster to + * have two call sites, with longest_match() inlined at + * each. + */ + if (unlikely(in_end - in_next < DEFLATE_MAX_MATCH_LEN)) { + max_len = in_end - in_next; + nice_len = MIN(nice_len, max_len); + } + next_len = hc_matchfinder_longest_match(&c->p.g.hc_mf, + &in_cur_base, + in_next, + cur_len, + max_len, + nice_len, + c->max_search_depth / 2, + next_hashes, + &next_offset); + in_next += 1; + + if (next_len > cur_len) { + /* Found a longer match at the next position. + * Output a literal. Then the next match + * becomes the current match. */ + deflate_choose_literal(c, *(in_next - 2), &litrunlen); + cur_len = next_len; + cur_offset = next_offset; + goto have_cur_match; + } + + /* No longer match at the next position. + * Output the current match. */ + deflate_choose_match(c, cur_len, cur_offset, + &litrunlen, &next_seq); + in_next = hc_matchfinder_skip_positions(&c->p.g.hc_mf, + &in_cur_base, + in_next, + in_end, + cur_len - 2, + next_hashes); + + /* Check if it's time to output another block. */ + } while (in_next < in_max_block_end && + !should_end_block(&c->split_stats, in_block_begin, in_next, in_end)); + + deflate_finish_sequence(next_seq, litrunlen); + deflate_flush_block(c, &os, in_block_begin, + in_next - in_block_begin, + in_next == in_end, false); + } while (in_next != in_end); + + return deflate_flush_output(&os); +} + +#if SUPPORT_NEAR_OPTIMAL_PARSING + +/* + * Follow the minimum-cost path in the graph of possible match/literal choices + * for the current block and compute the frequencies of the Huffman symbols that + * would be needed to output those matches and literals. + */ +static void +deflate_tally_item_list(struct libdeflate_compressor *c, u32 block_length) +{ + struct deflate_optimum_node *cur_node = &c->p.n.optimum_nodes[0]; + struct deflate_optimum_node *end_node = &c->p.n.optimum_nodes[block_length]; + do { + unsigned length = cur_node->item & OPTIMUM_LEN_MASK; + unsigned offset = cur_node->item >> OPTIMUM_OFFSET_SHIFT; + + if (length == 1) { + /* Literal */ + c->freqs.litlen[offset]++; + } else { + /* Match */ + c->freqs.litlen[257 + deflate_length_slot[length]]++; + c->freqs.offset[deflate_get_offset_slot(c, offset)]++; + } + cur_node += length; + } while (cur_node != end_node); +} + +/* Set the current cost model from the codeword lengths specified in @lens. */ +static void +deflate_set_costs_from_codes(struct libdeflate_compressor *c, + const struct deflate_lens *lens) +{ + unsigned i; + + /* Literals */ + for (i = 0; i < DEFLATE_NUM_LITERALS; i++) { + u32 bits = (lens->litlen[i] ? lens->litlen[i] : LITERAL_NOSTAT_BITS); + c->p.n.costs.literal[i] = bits << COST_SHIFT; + } + + /* Lengths */ + for (i = DEFLATE_MIN_MATCH_LEN; i <= DEFLATE_MAX_MATCH_LEN; i++) { + unsigned length_slot = deflate_length_slot[i]; + unsigned litlen_sym = 257 + length_slot; + u32 bits = (lens->litlen[litlen_sym] ? lens->litlen[litlen_sym] : LENGTH_NOSTAT_BITS); + bits += deflate_extra_length_bits[length_slot]; + c->p.n.costs.length[i] = bits << COST_SHIFT; + } + + /* Offset slots */ + for (i = 0; i < ARRAY_LEN(deflate_offset_slot_base); i++) { + u32 bits = (lens->offset[i] ? lens->offset[i] : OFFSET_NOSTAT_BITS); + bits += deflate_extra_offset_bits[i]; + c->p.n.costs.offset_slot[i] = bits << COST_SHIFT; + } +} + +static forceinline u32 +deflate_default_literal_cost(unsigned literal) +{ + STATIC_ASSERT(COST_SHIFT == 3); + /* 66 is 8.25 bits/symbol */ + return 66; +} + +static forceinline u32 +deflate_default_length_slot_cost(unsigned length_slot) +{ + STATIC_ASSERT(COST_SHIFT == 3); + /* 60 is 7.5 bits/symbol */ + return 60 + ((u32)deflate_extra_length_bits[length_slot] << COST_SHIFT); +} + +static forceinline u32 +deflate_default_offset_slot_cost(unsigned offset_slot) +{ + STATIC_ASSERT(COST_SHIFT == 3); + /* 39 is 4.875 bits/symbol */ + return 39 + ((u32)deflate_extra_offset_bits[offset_slot] << COST_SHIFT); +} + +/* + * Set default symbol costs for the first block's first optimization pass. + * + * It works well to assume that each symbol is equally probable. This results + * in each symbol being assigned a cost of (-log2(1.0/num_syms) * (1 << + * COST_SHIFT)) where 'num_syms' is the number of symbols in the corresponding + * alphabet. However, we intentionally bias the parse towards matches rather + * than literals by using a slightly lower default cost for length symbols than + * for literals. This often improves the compression ratio slightly. + */ +static void +deflate_set_default_costs(struct libdeflate_compressor *c) +{ + unsigned i; + + /* Literals */ + for (i = 0; i < DEFLATE_NUM_LITERALS; i++) + c->p.n.costs.literal[i] = deflate_default_literal_cost(i); + + /* Lengths */ + for (i = DEFLATE_MIN_MATCH_LEN; i <= DEFLATE_MAX_MATCH_LEN; i++) + c->p.n.costs.length[i] = deflate_default_length_slot_cost( + deflate_length_slot[i]); + + /* Offset slots */ + for (i = 0; i < ARRAY_LEN(deflate_offset_slot_base); i++) + c->p.n.costs.offset_slot[i] = deflate_default_offset_slot_cost(i); +} + +static forceinline void +deflate_adjust_cost(u32 *cost_p, u32 default_cost) +{ + *cost_p += ((s32)default_cost - (s32)*cost_p) >> 1; +} + +/* + * Adjust the costs when beginning a new block. + * + * Since the current costs have been optimized for the data, it's undesirable to + * throw them away and start over with the default costs. At the same time, we + * don't want to bias the parse by assuming that the next block will be similar + * to the current block. As a compromise, make the costs closer to the + * defaults, but don't simply set them to the defaults. + */ +static void +deflate_adjust_costs(struct libdeflate_compressor *c) +{ + unsigned i; + + /* Literals */ + for (i = 0; i < DEFLATE_NUM_LITERALS; i++) + deflate_adjust_cost(&c->p.n.costs.literal[i], + deflate_default_literal_cost(i)); + + /* Lengths */ + for (i = DEFLATE_MIN_MATCH_LEN; i <= DEFLATE_MAX_MATCH_LEN; i++) + deflate_adjust_cost(&c->p.n.costs.length[i], + deflate_default_length_slot_cost( + deflate_length_slot[i])); + + /* Offset slots */ + for (i = 0; i < ARRAY_LEN(deflate_offset_slot_base); i++) + deflate_adjust_cost(&c->p.n.costs.offset_slot[i], + deflate_default_offset_slot_cost(i)); +} + +/* + * Find the minimum-cost path through the graph of possible match/literal + * choices for this block. + * + * We find the minimum cost path from 'c->p.n.optimum_nodes[0]', which + * represents the node at the beginning of the block, to + * 'c->p.n.optimum_nodes[block_length]', which represents the node at the end of + * the block. Edge costs are evaluated using the cost model 'c->p.n.costs'. + * + * The algorithm works backwards, starting at the end node and proceeding + * backwards one node at a time. At each node, the minimum cost to reach the + * end node is computed and the match/literal choice that begins that path is + * saved. + */ +static void +deflate_find_min_cost_path(struct libdeflate_compressor *c, + const u32 block_length, + const struct lz_match *cache_ptr) +{ + struct deflate_optimum_node *end_node = &c->p.n.optimum_nodes[block_length]; + struct deflate_optimum_node *cur_node = end_node; + + cur_node->cost_to_end = 0; + do { + unsigned num_matches; + unsigned literal; + u32 best_cost_to_end; + + cur_node--; + cache_ptr--; + + num_matches = cache_ptr->length; + literal = cache_ptr->offset; + + /* It's always possible to choose a literal. */ + best_cost_to_end = c->p.n.costs.literal[literal] + + (cur_node + 1)->cost_to_end; + cur_node->item = ((u32)literal << OPTIMUM_OFFSET_SHIFT) | 1; + + /* Also consider matches if there are any. */ + if (num_matches) { + const struct lz_match *match; + unsigned len; + unsigned offset; + unsigned offset_slot; + u32 offset_cost; + u32 cost_to_end; + + /* + * Consider each length from the minimum + * (DEFLATE_MIN_MATCH_LEN) to the length of the longest + * match found at this position. For each length, we + * consider only the smallest offset for which that + * length is available. Although this is not guaranteed + * to be optimal due to the possibility of a larger + * offset costing less than a smaller offset to code, + * this is a very useful heuristic. + */ + match = cache_ptr - num_matches; + len = DEFLATE_MIN_MATCH_LEN; + do { + offset = match->offset; + offset_slot = deflate_get_offset_slot(c, offset); + offset_cost = c->p.n.costs.offset_slot[offset_slot]; + do { + cost_to_end = offset_cost + + c->p.n.costs.length[len] + + (cur_node + len)->cost_to_end; + if (cost_to_end < best_cost_to_end) { + best_cost_to_end = cost_to_end; + cur_node->item = ((u32)offset << OPTIMUM_OFFSET_SHIFT) | len; + } + } while (++len <= match->length); + } while (++match != cache_ptr); + cache_ptr -= num_matches; + } + cur_node->cost_to_end = best_cost_to_end; + } while (cur_node != &c->p.n.optimum_nodes[0]); +} + +/* + * Choose the literal/match sequence to use for the current block. The basic + * algorithm finds a minimum-cost path through the block's graph of + * literal/match choices, given a cost model. However, the cost of each symbol + * is unknown until the Huffman codes have been built, but at the same time the + * Huffman codes depend on the frequencies of chosen symbols. Consequently, + * multiple passes must be used to try to approximate an optimal solution. The + * first pass uses default costs, mixed with the costs from the previous block + * if any. Later passes use the Huffman codeword lengths from the previous pass + * as the costs. + */ +static void +deflate_optimize_block(struct libdeflate_compressor *c, u32 block_length, + const struct lz_match *cache_ptr, bool is_first_block) +{ + unsigned num_passes_remaining = c->p.n.num_optim_passes; + u32 i; + + /* Force the block to really end at the desired length, even if some + * matches extend beyond it. */ + for (i = block_length; i <= MIN(block_length - 1 + DEFLATE_MAX_MATCH_LEN, + ARRAY_LEN(c->p.n.optimum_nodes) - 1); i++) + c->p.n.optimum_nodes[i].cost_to_end = 0x80000000; + + /* Set the initial costs. */ + if (is_first_block) + deflate_set_default_costs(c); + else + deflate_adjust_costs(c); + + for (;;) { + /* Find the minimum cost path for this pass. */ + deflate_find_min_cost_path(c, block_length, cache_ptr); + + /* Compute frequencies of the chosen symbols. */ + deflate_reset_symbol_frequencies(c); + deflate_tally_item_list(c, block_length); + + if (--num_passes_remaining == 0) + break; + + /* At least one optimization pass remains; update the costs. */ + deflate_make_huffman_codes(&c->freqs, &c->codes); + deflate_set_costs_from_codes(c, &c->codes.lens); + } +} + +/* + * This is the "near-optimal" DEFLATE compressor. It computes the optimal + * representation of each DEFLATE block using a minimum-cost path search over + * the graph of possible match/literal choices for that block, assuming a + * certain cost for each Huffman symbol. + * + * For several reasons, the end result is not guaranteed to be optimal: + * + * - Nonoptimal choice of blocks + * - Heuristic limitations on which matches are actually considered + * - Symbol costs are unknown until the symbols have already been chosen + * (so iterative optimization must be used) + */ +static size_t +deflate_compress_near_optimal(struct libdeflate_compressor * restrict c, + const u8 * restrict in, size_t in_nbytes, + u8 * restrict out, size_t out_nbytes_avail) +{ + const u8 *in_next = in; + const u8 *in_end = in_next + in_nbytes; + struct deflate_output_bitstream os; + const u8 *in_cur_base = in_next; + const u8 *in_next_slide = in_next + MIN(in_end - in_next, MATCHFINDER_WINDOW_SIZE); + unsigned max_len = DEFLATE_MAX_MATCH_LEN; + unsigned nice_len = MIN(c->nice_match_length, max_len); + u32 next_hashes[2] = {0, 0}; + + deflate_init_output(&os, out, out_nbytes_avail); + bt_matchfinder_init(&c->p.n.bt_mf); + + do { + /* Starting a new DEFLATE block. */ + + struct lz_match *cache_ptr = c->p.n.match_cache; + const u8 * const in_block_begin = in_next; + const u8 * const in_max_block_end = + in_next + MIN(in_end - in_next, SOFT_MAX_BLOCK_LENGTH); + const u8 *next_observation = in_next; + + init_block_split_stats(&c->split_stats); + + /* + * Find matches until we decide to end the block. We end the + * block if any of the following is true: + * + * (1) Maximum block length has been reached + * (2) Match catch may overflow. + * (3) Block split heuristic says to split now. + */ + do { + struct lz_match *matches; + unsigned best_len; + + /* Slide the window forward if needed. */ + if (in_next == in_next_slide) { + bt_matchfinder_slide_window(&c->p.n.bt_mf); + in_cur_base = in_next; + in_next_slide = in_next + MIN(in_end - in_next, + MATCHFINDER_WINDOW_SIZE); + } + + /* Decrease the maximum and nice match lengths if we're + * approaching the end of the input buffer. */ + if (unlikely(max_len > in_end - in_next)) { + max_len = in_end - in_next; + nice_len = MIN(nice_len, max_len); + } + + /* + * Find matches with the current position using the + * binary tree matchfinder and save them in + * 'match_cache'. + * + * Note: the binary tree matchfinder is more suited for + * optimal parsing than the hash chain matchfinder. The + * reasons for this include: + * + * - The binary tree matchfinder can find more matches + * in the same number of steps. + * - One of the major advantages of hash chains is that + * skipping positions (not searching for matches at + * them) is faster; however, with optimal parsing we + * search for matches at almost all positions, so this + * advantage of hash chains is negated. + */ + matches = cache_ptr; + best_len = 0; + if (likely(max_len >= BT_MATCHFINDER_REQUIRED_NBYTES)) { + cache_ptr = bt_matchfinder_get_matches(&c->p.n.bt_mf, + in_cur_base, + in_next - in_cur_base, + max_len, + nice_len, + c->max_search_depth, + next_hashes, + &best_len, + matches); + } + + if (in_next >= next_observation) { + if (best_len >= 4) { + observe_match(&c->split_stats, best_len); + next_observation = in_next + best_len; + } else { + observe_literal(&c->split_stats, *in_next); + next_observation = in_next + 1; + } + } + + cache_ptr->length = cache_ptr - matches; + cache_ptr->offset = *in_next; + in_next++; + cache_ptr++; + + /* + * If there was a very long match found, don't cache any + * matches for the bytes covered by that match. This + * avoids degenerate behavior when compressing highly + * redundant data, where the number of matches can be + * very large. + * + * This heuristic doesn't actually hurt the compression + * ratio very much. If there's a long match, then the + * data must be highly compressible, so it doesn't + * matter much what we do. + */ + if (best_len >= DEFLATE_MIN_MATCH_LEN && best_len >= nice_len) { + --best_len; + do { + if (in_next == in_next_slide) { + bt_matchfinder_slide_window(&c->p.n.bt_mf); + in_cur_base = in_next; + in_next_slide = in_next + MIN(in_end - in_next, + MATCHFINDER_WINDOW_SIZE); + } + if (unlikely(max_len > in_end - in_next)) { + max_len = in_end - in_next; + nice_len = MIN(nice_len, max_len); + } + if (max_len >= BT_MATCHFINDER_REQUIRED_NBYTES) { + bt_matchfinder_skip_position(&c->p.n.bt_mf, + in_cur_base, + in_next - in_cur_base, + nice_len, + c->max_search_depth, + next_hashes); + } + cache_ptr->length = 0; + cache_ptr->offset = *in_next; + in_next++; + cache_ptr++; + } while (--best_len); + } + } while (in_next < in_max_block_end && + cache_ptr < &c->p.n.match_cache[CACHE_LENGTH] && + !should_end_block(&c->split_stats, in_block_begin, in_next, in_end)); + + /* All the matches for this block have been cached. Now choose + * the sequence of items to output and flush the block. */ + deflate_optimize_block(c, in_next - in_block_begin, cache_ptr, + in_block_begin == in); + deflate_flush_block(c, &os, in_block_begin, in_next - in_block_begin, + in_next == in_end, true); + } while (in_next != in_end); + + return deflate_flush_output(&os); +} + +#endif /* SUPPORT_NEAR_OPTIMAL_PARSING */ + +/* Initialize c->offset_slot_fast. */ +static void +deflate_init_offset_slot_fast(struct libdeflate_compressor *c) +{ + unsigned offset_slot; + unsigned offset; + unsigned offset_end; + + for (offset_slot = 0; + offset_slot < ARRAY_LEN(deflate_offset_slot_base); + offset_slot++) + { + offset = deflate_offset_slot_base[offset_slot]; + #if USE_FULL_OFFSET_SLOT_FAST + offset_end = offset + (1 << deflate_extra_offset_bits[offset_slot]); + do { + c->offset_slot_fast[offset] = offset_slot; + } while (++offset != offset_end); + #else + if (offset <= 256) { + offset_end = offset + (1 << deflate_extra_offset_bits[offset_slot]); + do { + c->offset_slot_fast[offset - 1] = offset_slot; + } while (++offset != offset_end); + } else { + offset_end = offset + (1 << deflate_extra_offset_bits[offset_slot]); + do { + c->offset_slot_fast[256 + ((offset - 1) >> 7)] = offset_slot; + } while ((offset += (1 << 7)) != offset_end); + } + #endif + } +} + +LIBDEFLATEEXPORT struct libdeflate_compressor * LIBDEFLATEAPI +libdeflate_alloc_compressor(int compression_level) +{ + struct libdeflate_compressor *c; + size_t size = offsetof(struct libdeflate_compressor, p); + + if (compression_level < 0 || compression_level > 12) + return NULL; + +#if SUPPORT_NEAR_OPTIMAL_PARSING + if (compression_level >= 8) + size += sizeof(c->p.n); + else if (compression_level >= 1) + size += sizeof(c->p.g); +#else + if (compression_level >= 1) + size += sizeof(c->p.g); +#endif + + c = libdeflate_aligned_malloc(MATCHFINDER_MEM_ALIGNMENT, size); + if (!c) + return NULL; + + c->compression_level = compression_level; + + /* + * The higher the compression level, the more we should bother trying to + * compress very small inputs. + */ + c->min_size_to_compress = 56 - (compression_level * 4); + + switch (compression_level) { + case 0: + c->impl = deflate_compress_none; + break; + case 1: + c->impl = deflate_compress_greedy; + c->max_search_depth = 2; + c->nice_match_length = 8; + break; + case 2: + c->impl = deflate_compress_greedy; + c->max_search_depth = 6; + c->nice_match_length = 10; + break; + case 3: + c->impl = deflate_compress_greedy; + c->max_search_depth = 12; + c->nice_match_length = 14; + break; + case 4: + c->impl = deflate_compress_greedy; + c->max_search_depth = 24; + c->nice_match_length = 24; + break; + case 5: + c->impl = deflate_compress_lazy; + c->max_search_depth = 20; + c->nice_match_length = 30; + break; + case 6: + c->impl = deflate_compress_lazy; + c->max_search_depth = 40; + c->nice_match_length = 65; + break; + case 7: + c->impl = deflate_compress_lazy; + c->max_search_depth = 100; + c->nice_match_length = 130; + break; +#if SUPPORT_NEAR_OPTIMAL_PARSING + case 8: + c->impl = deflate_compress_near_optimal; + c->max_search_depth = 12; + c->nice_match_length = 20; + c->p.n.num_optim_passes = 1; + break; + case 9: + c->impl = deflate_compress_near_optimal; + c->max_search_depth = 16; + c->nice_match_length = 26; + c->p.n.num_optim_passes = 2; + break; + case 10: + c->impl = deflate_compress_near_optimal; + c->max_search_depth = 30; + c->nice_match_length = 50; + c->p.n.num_optim_passes = 2; + break; + case 11: + c->impl = deflate_compress_near_optimal; + c->max_search_depth = 60; + c->nice_match_length = 80; + c->p.n.num_optim_passes = 3; + break; + default: + c->impl = deflate_compress_near_optimal; + c->max_search_depth = 100; + c->nice_match_length = 133; + c->p.n.num_optim_passes = 4; + break; +#else + case 8: + c->impl = deflate_compress_lazy; + c->max_search_depth = 150; + c->nice_match_length = 200; + break; + default: + c->impl = deflate_compress_lazy; + c->max_search_depth = 200; + c->nice_match_length = DEFLATE_MAX_MATCH_LEN; + break; +#endif + } + + deflate_init_offset_slot_fast(c); + deflate_init_static_codes(c); + + return c; +} + +LIBDEFLATEEXPORT size_t LIBDEFLATEAPI +libdeflate_deflate_compress(struct libdeflate_compressor *c, + const void *in, size_t in_nbytes, + void *out, size_t out_nbytes_avail) +{ + if (unlikely(out_nbytes_avail < OUTPUT_END_PADDING)) + return 0; + + /* For extremely small inputs just use a single uncompressed block. */ + if (unlikely(in_nbytes < c->min_size_to_compress)) { + struct deflate_output_bitstream os; + deflate_init_output(&os, out, out_nbytes_avail); + if (in_nbytes == 0) + in = &os; /* Avoid passing NULL to memcpy() */ + deflate_write_uncompressed_block(&os, in, in_nbytes, true); + return deflate_flush_output(&os); + } + + return (*c->impl)(c, in, in_nbytes, out, out_nbytes_avail); +} + +LIBDEFLATEEXPORT void LIBDEFLATEAPI +libdeflate_free_compressor(struct libdeflate_compressor *c) +{ + libdeflate_aligned_free(c); +} + +unsigned int +deflate_get_compression_level(struct libdeflate_compressor *c) +{ + return c->compression_level; +} + +LIBDEFLATEEXPORT size_t LIBDEFLATEAPI +libdeflate_deflate_compress_bound(struct libdeflate_compressor *c, + size_t in_nbytes) +{ + /* + * The worst case is all uncompressed blocks where one block has length + * <= MIN_BLOCK_LENGTH and the others have length MIN_BLOCK_LENGTH. + * Each uncompressed block has 5 bytes of overhead: 1 for BFINAL, BTYPE, + * and alignment to a byte boundary; 2 for LEN; and 2 for NLEN. + */ + size_t max_num_blocks = MAX(DIV_ROUND_UP(in_nbytes, MIN_BLOCK_LENGTH), 1); + return (5 * max_num_blocks) + in_nbytes + 1 + OUTPUT_END_PADDING; +} diff --git a/ext/libdeflate/lib/deflate_compress.h b/ext/libdeflate/lib/deflate_compress.h new file mode 100644 index 0000000000..d97d019ae7 --- /dev/null +++ b/ext/libdeflate/lib/deflate_compress.h @@ -0,0 +1,13 @@ +#ifndef LIB_DEFLATE_COMPRESS_H +#define LIB_DEFLATE_COMPRESS_H + +#include "lib_common.h" + +/* DEFLATE compression is private to deflate_compress.c, but we do need to be + * able to query the compression level for zlib and gzip header generation. */ + +struct libdeflate_compressor; + +unsigned int deflate_get_compression_level(struct libdeflate_compressor *c); + +#endif /* LIB_DEFLATE_COMPRESS_H */ diff --git a/ext/libdeflate/lib/deflate_constants.h b/ext/libdeflate/lib/deflate_constants.h new file mode 100644 index 0000000000..a10b57dec0 --- /dev/null +++ b/ext/libdeflate/lib/deflate_constants.h @@ -0,0 +1,66 @@ +/* + * deflate_constants.h - constants for the DEFLATE compression format + */ + +#ifndef LIB_DEFLATE_CONSTANTS_H +#define LIB_DEFLATE_CONSTANTS_H + +/* Valid block types */ +#define DEFLATE_BLOCKTYPE_UNCOMPRESSED 0 +#define DEFLATE_BLOCKTYPE_STATIC_HUFFMAN 1 +#define DEFLATE_BLOCKTYPE_DYNAMIC_HUFFMAN 2 + +/* Minimum and maximum supported match lengths (in bytes) */ +#define DEFLATE_MIN_MATCH_LEN 3 +#define DEFLATE_MAX_MATCH_LEN 258 + +/* Minimum and maximum supported match offsets (in bytes) */ +#define DEFLATE_MIN_MATCH_OFFSET 1 +#define DEFLATE_MAX_MATCH_OFFSET 32768 + +#define DEFLATE_MAX_WINDOW_SIZE 32768 + +/* Number of symbols in each Huffman code. Note: for the literal/length + * and offset codes, these are actually the maximum values; a given block + * might use fewer symbols. */ +#define DEFLATE_NUM_PRECODE_SYMS 19 +#define DEFLATE_NUM_LITLEN_SYMS 288 +#define DEFLATE_NUM_OFFSET_SYMS 32 + +/* The maximum number of symbols across all codes */ +#define DEFLATE_MAX_NUM_SYMS 288 + +/* Division of symbols in the literal/length code */ +#define DEFLATE_NUM_LITERALS 256 +#define DEFLATE_END_OF_BLOCK 256 +#define DEFLATE_NUM_LEN_SYMS 31 + +/* Maximum codeword length, in bits, within each Huffman code */ +#define DEFLATE_MAX_PRE_CODEWORD_LEN 7 +#define DEFLATE_MAX_LITLEN_CODEWORD_LEN 15 +#define DEFLATE_MAX_OFFSET_CODEWORD_LEN 15 + +/* The maximum codeword length across all codes */ +#define DEFLATE_MAX_CODEWORD_LEN 15 + +/* Maximum possible overrun when decoding codeword lengths */ +#define DEFLATE_MAX_LENS_OVERRUN 137 + +/* + * Maximum number of extra bits that may be required to represent a match + * length or offset. + * + * TODO: are we going to have full DEFLATE64 support? If so, up to 16 + * length bits must be supported. + */ +#define DEFLATE_MAX_EXTRA_LENGTH_BITS 5 +#define DEFLATE_MAX_EXTRA_OFFSET_BITS 14 + +/* The maximum number of bits in which a match can be represented. This + * is the absolute worst case, which assumes the longest possible Huffman + * codewords and the maximum numbers of extra bits. */ +#define DEFLATE_MAX_MATCH_BITS \ + (DEFLATE_MAX_LITLEN_CODEWORD_LEN + DEFLATE_MAX_EXTRA_LENGTH_BITS + \ + DEFLATE_MAX_OFFSET_CODEWORD_LEN + DEFLATE_MAX_EXTRA_OFFSET_BITS) + +#endif /* LIB_DEFLATE_CONSTANTS_H */ diff --git a/ext/libdeflate/lib/hc_matchfinder.h b/ext/libdeflate/lib/hc_matchfinder.h new file mode 100644 index 0000000000..becf844fa7 --- /dev/null +++ b/ext/libdeflate/lib/hc_matchfinder.h @@ -0,0 +1,407 @@ +/* + * hc_matchfinder.h - Lempel-Ziv matchfinding with a hash table of linked lists + * + * Originally public domain; changes after 2016-09-07 are copyrighted. + * + * Copyright 2016 Eric Biggers + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * --------------------------------------------------------------------------- + * + * Algorithm + * + * This is a Hash Chains (hc) based matchfinder. + * + * The main data structure is a hash table where each hash bucket contains a + * linked list (or "chain") of sequences whose first 4 bytes share the same hash + * code. Each sequence is identified by its starting position in the input + * buffer. + * + * The algorithm processes the input buffer sequentially. At each byte + * position, the hash code of the first 4 bytes of the sequence beginning at + * that position (the sequence being matched against) is computed. This + * identifies the hash bucket to use for that position. Then, this hash + * bucket's linked list is searched for matches. Then, a new linked list node + * is created to represent the current sequence and is prepended to the list. + * + * This algorithm has several useful properties: + * + * - It only finds true Lempel-Ziv matches; i.e., those where the matching + * sequence occurs prior to the sequence being matched against. + * + * - The sequences in each linked list are always sorted by decreasing starting + * position. Therefore, the closest (smallest offset) matches are found + * first, which in many compression formats tend to be the cheapest to encode. + * + * - Although fast running time is not guaranteed due to the possibility of the + * lists getting very long, the worst degenerate behavior can be easily + * prevented by capping the number of nodes searched at each position. + * + * - If the compressor decides not to search for matches at a certain position, + * then that position can be quickly inserted without searching the list. + * + * - The algorithm is adaptable to sliding windows: just store the positions + * relative to a "base" value that is updated from time to time, and stop + * searching each list when the sequences get too far away. + * + * ---------------------------------------------------------------------------- + * + * Optimizations + * + * The main hash table and chains handle length 4+ matches. Length 3 matches + * are handled by a separate hash table with no chains. This works well for + * typical "greedy" or "lazy"-style compressors, where length 3 matches are + * often only helpful if they have small offsets. Instead of searching a full + * chain for length 3+ matches, the algorithm just checks for one close length 3 + * match, then focuses on finding length 4+ matches. + * + * The longest_match() and skip_positions() functions are inlined into the + * compressors that use them. This isn't just about saving the overhead of a + * function call. These functions are intended to be called from the inner + * loops of compressors, where giving the compiler more control over register + * allocation is very helpful. There is also significant benefit to be gained + * from allowing the CPU to predict branches independently at each call site. + * For example, "lazy"-style compressors can be written with two calls to + * longest_match(), each of which starts with a different 'best_len' and + * therefore has significantly different performance characteristics. + * + * Although any hash function can be used, a multiplicative hash is fast and + * works well. + * + * On some processors, it is significantly faster to extend matches by whole + * words (32 or 64 bits) instead of by individual bytes. For this to be the + * case, the processor must implement unaligned memory accesses efficiently and + * must have either a fast "find first set bit" instruction or a fast "find last + * set bit" instruction, depending on the processor's endianness. + * + * The code uses one loop for finding the first match and one loop for finding a + * longer match. Each of these loops is tuned for its respective task and in + * combination are faster than a single generalized loop that handles both + * tasks. + * + * The code also uses a tight inner loop that only compares the last and first + * bytes of a potential match. It is only when these bytes match that a full + * match extension is attempted. + * + * ---------------------------------------------------------------------------- + */ + +#include "matchfinder_common.h" + +#define HC_MATCHFINDER_HASH3_ORDER 15 +#define HC_MATCHFINDER_HASH4_ORDER 16 + +#define HC_MATCHFINDER_TOTAL_HASH_SIZE \ + (((1UL << HC_MATCHFINDER_HASH3_ORDER) + \ + (1UL << HC_MATCHFINDER_HASH4_ORDER)) * sizeof(mf_pos_t)) + +struct hc_matchfinder { + + /* The hash table for finding length 3 matches */ + mf_pos_t hash3_tab[1UL << HC_MATCHFINDER_HASH3_ORDER]; + + /* The hash table which contains the first nodes of the linked lists for + * finding length 4+ matches */ + mf_pos_t hash4_tab[1UL << HC_MATCHFINDER_HASH4_ORDER]; + + /* The "next node" references for the linked lists. The "next node" of + * the node for the sequence with position 'pos' is 'next_tab[pos]'. */ + mf_pos_t next_tab[MATCHFINDER_WINDOW_SIZE]; + +} +#ifdef _aligned_attribute + _aligned_attribute(MATCHFINDER_MEM_ALIGNMENT) +#endif +; + +/* Prepare the matchfinder for a new input buffer. */ +static forceinline void +hc_matchfinder_init(struct hc_matchfinder *mf) +{ + STATIC_ASSERT(HC_MATCHFINDER_TOTAL_HASH_SIZE % + MATCHFINDER_SIZE_ALIGNMENT == 0); + + matchfinder_init((mf_pos_t *)mf, HC_MATCHFINDER_TOTAL_HASH_SIZE); +} + +static forceinline void +hc_matchfinder_slide_window(struct hc_matchfinder *mf) +{ + STATIC_ASSERT(sizeof(*mf) % MATCHFINDER_SIZE_ALIGNMENT == 0); + + matchfinder_rebase((mf_pos_t *)mf, sizeof(*mf)); +} + +/* + * Find the longest match longer than 'best_len' bytes. + * + * @mf + * The matchfinder structure. + * @in_base_p + * Location of a pointer which points to the place in the input data the + * matchfinder currently stores positions relative to. This may be updated + * by this function. + * @cur_pos + * The current position in the input buffer relative to @in_base (the + * position of the sequence being matched against). + * @best_len + * Require a match longer than this length. + * @max_len + * The maximum permissible match length at this position. + * @nice_len + * Stop searching if a match of at least this length is found. + * Must be <= @max_len. + * @max_search_depth + * Limit on the number of potential matches to consider. Must be >= 1. + * @next_hashes + * The precomputed hash codes for the sequence beginning at @in_next. + * These will be used and then updated with the precomputed hashcodes for + * the sequence beginning at @in_next + 1. + * @offset_ret + * If a match is found, its offset is returned in this location. + * + * Return the length of the match found, or 'best_len' if no match longer than + * 'best_len' was found. + */ +static forceinline u32 +hc_matchfinder_longest_match(struct hc_matchfinder * const restrict mf, + const u8 ** const restrict in_base_p, + const u8 * const restrict in_next, + u32 best_len, + const u32 max_len, + const u32 nice_len, + const u32 max_search_depth, + u32 * const restrict next_hashes, + u32 * const restrict offset_ret) +{ + u32 depth_remaining = max_search_depth; + const u8 *best_matchptr = in_next; + mf_pos_t cur_node3, cur_node4; + u32 hash3, hash4; + u32 next_hashseq; + u32 seq4; + const u8 *matchptr; + u32 len; + u32 cur_pos = in_next - *in_base_p; + const u8 *in_base; + mf_pos_t cutoff; + + if (cur_pos == MATCHFINDER_WINDOW_SIZE) { + hc_matchfinder_slide_window(mf); + *in_base_p += MATCHFINDER_WINDOW_SIZE; + cur_pos = 0; + } + + in_base = *in_base_p; + cutoff = cur_pos - MATCHFINDER_WINDOW_SIZE; + + if (unlikely(max_len < 5)) /* can we read 4 bytes from 'in_next + 1'? */ + goto out; + + /* Get the precomputed hash codes. */ + hash3 = next_hashes[0]; + hash4 = next_hashes[1]; + + /* From the hash buckets, get the first node of each linked list. */ + cur_node3 = mf->hash3_tab[hash3]; + cur_node4 = mf->hash4_tab[hash4]; + + /* Update for length 3 matches. This replaces the singleton node in the + * 'hash3' bucket with the node for the current sequence. */ + mf->hash3_tab[hash3] = cur_pos; + + /* Update for length 4 matches. This prepends the node for the current + * sequence to the linked list in the 'hash4' bucket. */ + mf->hash4_tab[hash4] = cur_pos; + mf->next_tab[cur_pos] = cur_node4; + + /* Compute the next hash codes. */ + next_hashseq = get_unaligned_le32(in_next + 1); + next_hashes[0] = lz_hash(next_hashseq & 0xFFFFFF, HC_MATCHFINDER_HASH3_ORDER); + next_hashes[1] = lz_hash(next_hashseq, HC_MATCHFINDER_HASH4_ORDER); + prefetchw(&mf->hash3_tab[next_hashes[0]]); + prefetchw(&mf->hash4_tab[next_hashes[1]]); + + if (best_len < 4) { /* No match of length >= 4 found yet? */ + + /* Check for a length 3 match if needed. */ + + if (cur_node3 <= cutoff) + goto out; + + seq4 = load_u32_unaligned(in_next); + + if (best_len < 3) { + matchptr = &in_base[cur_node3]; + if (load_u24_unaligned(matchptr) == loaded_u32_to_u24(seq4)) { + best_len = 3; + best_matchptr = matchptr; + } + } + + /* Check for a length 4 match. */ + + if (cur_node4 <= cutoff) + goto out; + + for (;;) { + /* No length 4 match found yet. Check the first 4 bytes. */ + matchptr = &in_base[cur_node4]; + + if (load_u32_unaligned(matchptr) == seq4) + break; + + /* The first 4 bytes did not match. Keep trying. */ + cur_node4 = mf->next_tab[cur_node4 & (MATCHFINDER_WINDOW_SIZE - 1)]; + if (cur_node4 <= cutoff || !--depth_remaining) + goto out; + } + + /* Found a match of length >= 4. Extend it to its full length. */ + best_matchptr = matchptr; + best_len = lz_extend(in_next, best_matchptr, 4, max_len); + if (best_len >= nice_len) + goto out; + cur_node4 = mf->next_tab[cur_node4 & (MATCHFINDER_WINDOW_SIZE - 1)]; + if (cur_node4 <= cutoff || !--depth_remaining) + goto out; + } else { + if (cur_node4 <= cutoff || best_len >= nice_len) + goto out; + } + + /* Check for matches of length >= 5. */ + + for (;;) { + for (;;) { + matchptr = &in_base[cur_node4]; + + /* Already found a length 4 match. Try for a longer + * match; start by checking either the last 4 bytes and + * the first 4 bytes, or the last byte. (The last byte, + * the one which would extend the match length by 1, is + * the most important.) */ + #if UNALIGNED_ACCESS_IS_FAST + if ((load_u32_unaligned(matchptr + best_len - 3) == + load_u32_unaligned(in_next + best_len - 3)) && + (load_u32_unaligned(matchptr) == + load_u32_unaligned(in_next))) + #else + if (matchptr[best_len] == in_next[best_len]) + #endif + break; + + /* Continue to the next node in the list. */ + cur_node4 = mf->next_tab[cur_node4 & (MATCHFINDER_WINDOW_SIZE - 1)]; + if (cur_node4 <= cutoff || !--depth_remaining) + goto out; + } + + #if UNALIGNED_ACCESS_IS_FAST + len = 4; + #else + len = 0; + #endif + len = lz_extend(in_next, matchptr, len, max_len); + if (len > best_len) { + /* This is the new longest match. */ + best_len = len; + best_matchptr = matchptr; + if (best_len >= nice_len) + goto out; + } + + /* Continue to the next node in the list. */ + cur_node4 = mf->next_tab[cur_node4 & (MATCHFINDER_WINDOW_SIZE - 1)]; + if (cur_node4 <= cutoff || !--depth_remaining) + goto out; + } +out: + *offset_ret = in_next - best_matchptr; + return best_len; +} + +/* + * Advance the matchfinder, but don't search for matches. + * + * @mf + * The matchfinder structure. + * @in_base_p + * Location of a pointer which points to the place in the input data the + * matchfinder currently stores positions relative to. This may be updated + * by this function. + * @cur_pos + * The current position in the input buffer relative to @in_base. + * @end_pos + * The end position of the input buffer, relative to @in_base. + * @next_hashes + * The precomputed hash codes for the sequence beginning at @in_next. + * These will be used and then updated with the precomputed hashcodes for + * the sequence beginning at @in_next + @count. + * @count + * The number of bytes to advance. Must be > 0. + * + * Returns @in_next + @count. + */ +static forceinline const u8 * +hc_matchfinder_skip_positions(struct hc_matchfinder * const restrict mf, + const u8 ** const restrict in_base_p, + const u8 *in_next, + const u8 * const in_end, + const u32 count, + u32 * const restrict next_hashes) +{ + u32 cur_pos; + u32 hash3, hash4; + u32 next_hashseq; + u32 remaining = count; + + if (unlikely(count + 5 > in_end - in_next)) + return &in_next[count]; + + cur_pos = in_next - *in_base_p; + hash3 = next_hashes[0]; + hash4 = next_hashes[1]; + do { + if (cur_pos == MATCHFINDER_WINDOW_SIZE) { + hc_matchfinder_slide_window(mf); + *in_base_p += MATCHFINDER_WINDOW_SIZE; + cur_pos = 0; + } + mf->hash3_tab[hash3] = cur_pos; + mf->next_tab[cur_pos] = mf->hash4_tab[hash4]; + mf->hash4_tab[hash4] = cur_pos; + + next_hashseq = get_unaligned_le32(++in_next); + hash3 = lz_hash(next_hashseq & 0xFFFFFF, HC_MATCHFINDER_HASH3_ORDER); + hash4 = lz_hash(next_hashseq, HC_MATCHFINDER_HASH4_ORDER); + cur_pos++; + } while (--remaining); + + prefetchw(&mf->hash3_tab[hash3]); + prefetchw(&mf->hash4_tab[hash4]); + next_hashes[0] = hash3; + next_hashes[1] = hash4; + + return in_next; +} diff --git a/ext/libdeflate/lib/lib_common.h b/ext/libdeflate/lib/lib_common.h new file mode 100644 index 0000000000..2eea56c729 --- /dev/null +++ b/ext/libdeflate/lib/lib_common.h @@ -0,0 +1,67 @@ +/* + * lib_common.h - internal header included by all library code + */ + +#ifndef LIB_LIB_COMMON_H +#define LIB_LIB_COMMON_H + +#ifdef LIBDEFLATE_H +# error "lib_common.h must always be included before libdeflate.h" + /* because BUILDING_LIBDEFLATE must be set first */ +#endif + +#define BUILDING_LIBDEFLATE + +#include "../common/common_defs.h" + +/* + * Prefix with "_libdeflate_" all global symbols which are not part of the API + * and don't already have a "libdeflate" prefix. This avoids exposing overly + * generic names when libdeflate is built as a static library. + * + * Note that the chosen prefix is not really important and can be changed + * without breaking library users. It was just chosen so that the resulting + * symbol names are unlikely to conflict with those from any other software. + * Also note that this fixup has no useful effect when libdeflate is built as a + * shared library, since these symbols are not exported. + */ +#define SYM_FIXUP(sym) _libdeflate_##sym +#define deflate_get_compression_level SYM_FIXUP(deflate_get_compression_level) +#define _cpu_features SYM_FIXUP(_cpu_features) +#define setup_cpu_features SYM_FIXUP(setup_cpu_features) + +void *libdeflate_malloc(size_t size); +void libdeflate_free(void *ptr); + +void *libdeflate_aligned_malloc(size_t alignment, size_t size); +void libdeflate_aligned_free(void *ptr); + +#ifdef FREESTANDING +/* + * With -ffreestanding, may be missing, and we must provide + * implementations of memset(), memcpy(), memmove(), and memcmp(). + * See https://gcc.gnu.org/onlinedocs/gcc/Standards.html + * + * Also, -ffreestanding disables interpreting calls to these functions as + * built-ins. E.g., calling memcpy(&v, p, WORDBYTES) will make a function call, + * not be optimized to a single load instruction. For performance reasons we + * don't want that. So, declare these functions as macros that expand to the + * corresponding built-ins. This approach is recommended in the gcc man page. + * We still need the actual function definitions in case gcc calls them. + */ +void *memset(void *s, int c, size_t n); +#define memset(s, c, n) __builtin_memset((s), (c), (n)) + +void *memcpy(void *dest, const void *src, size_t n); +#define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n)) + +void *memmove(void *dest, const void *src, size_t n); +#define memmove(dest, src, n) __builtin_memmove((dest), (src), (n)) + +int memcmp(const void *s1, const void *s2, size_t n); +#define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n)) +#else +#include +#endif + +#endif /* LIB_LIB_COMMON_H */ diff --git a/ext/libdeflate/lib/matchfinder_common.h b/ext/libdeflate/lib/matchfinder_common.h new file mode 100644 index 0000000000..49ff3343e1 --- /dev/null +++ b/ext/libdeflate/lib/matchfinder_common.h @@ -0,0 +1,176 @@ +/* + * matchfinder_common.h - common code for Lempel-Ziv matchfinding + */ + +#ifndef LIB_MATCHFINDER_COMMON_H +#define LIB_MATCHFINDER_COMMON_H + +#include "lib_common.h" +#include "unaligned.h" + +#ifndef MATCHFINDER_WINDOW_ORDER +# error "MATCHFINDER_WINDOW_ORDER must be defined!" +#endif + +#define MATCHFINDER_WINDOW_SIZE (1UL << MATCHFINDER_WINDOW_ORDER) + +typedef s16 mf_pos_t; + +#define MATCHFINDER_INITVAL ((mf_pos_t)-MATCHFINDER_WINDOW_SIZE) + +/* + * Required alignment of the matchfinder buffer pointer and size. The values + * here come from the AVX-2 implementation, which is the worst case. + */ +#define MATCHFINDER_MEM_ALIGNMENT 32 +#define MATCHFINDER_SIZE_ALIGNMENT 128 + +#undef matchfinder_init +#undef matchfinder_rebase +#ifdef _aligned_attribute +# if defined(__arm__) || defined(__aarch64__) +# include "arm/matchfinder_impl.h" +# elif defined(__i386__) || defined(__x86_64__) +# include "x86/matchfinder_impl.h" +# endif +#endif + +/* + * Initialize the hash table portion of the matchfinder. + * + * Essentially, this is an optimized memset(). + * + * 'data' must be aligned to a MATCHFINDER_MEM_ALIGNMENT boundary, and + * 'size' must be a multiple of MATCHFINDER_SIZE_ALIGNMENT. + */ +#ifndef matchfinder_init +static forceinline void +matchfinder_init(mf_pos_t *data, size_t size) +{ + size_t num_entries = size / sizeof(*data); + size_t i; + + for (i = 0; i < num_entries; i++) + data[i] = MATCHFINDER_INITVAL; +} +#endif + +/* + * Slide the matchfinder by WINDOW_SIZE bytes. + * + * This must be called just after each WINDOW_SIZE bytes have been run through + * the matchfinder. + * + * This will subtract WINDOW_SIZE bytes from each entry in the array specified. + * The effect is that all entries are updated to be relative to the current + * position, rather than the position WINDOW_SIZE bytes prior. + * + * Underflow is detected and replaced with signed saturation. This ensures that + * once the sliding window has passed over a position, that position forever + * remains out of bounds. + * + * The array passed in must contain all matchfinder data that is + * position-relative. Concretely, this will include the hash table as well as + * the table of positions that is used to link together the sequences in each + * hash bucket. Note that in the latter table, the links are 1-ary in the case + * of "hash chains", and 2-ary in the case of "binary trees". In either case, + * the links need to be rebased in the same way. + * + * 'data' must be aligned to a MATCHFINDER_MEM_ALIGNMENT boundary, and + * 'size' must be a multiple of MATCHFINDER_SIZE_ALIGNMENT. + */ +#ifndef matchfinder_rebase +static forceinline void +matchfinder_rebase(mf_pos_t *data, size_t size) +{ + size_t num_entries = size / sizeof(*data); + size_t i; + + if (MATCHFINDER_WINDOW_SIZE == 32768) { + /* Branchless version for 32768 byte windows. If the value was + * already negative, clear all bits except the sign bit; this + * changes the value to -32768. Otherwise, set the sign bit; + * this is equivalent to subtracting 32768. */ + for (i = 0; i < num_entries; i++) { + u16 v = data[i]; + u16 sign_bit = v & 0x8000; + v &= sign_bit - ((sign_bit >> 15) ^ 1); + v |= 0x8000; + data[i] = v; + } + return; + } + + for (i = 0; i < num_entries; i++) { + if (data[i] >= 0) + data[i] -= (mf_pos_t)-MATCHFINDER_WINDOW_SIZE; + else + data[i] = (mf_pos_t)-MATCHFINDER_WINDOW_SIZE; + } +} +#endif + +/* + * The hash function: given a sequence prefix held in the low-order bits of a + * 32-bit value, multiply by a carefully-chosen large constant. Discard any + * bits of the product that don't fit in a 32-bit value, but take the + * next-highest @num_bits bits of the product as the hash value, as those have + * the most randomness. + */ +static forceinline u32 +lz_hash(u32 seq, unsigned num_bits) +{ + return (u32)(seq * 0x1E35A7BD) >> (32 - num_bits); +} + +/* + * Return the number of bytes at @matchptr that match the bytes at @strptr, up + * to a maximum of @max_len. Initially, @start_len bytes are matched. + */ +static forceinline unsigned +lz_extend(const u8 * const strptr, const u8 * const matchptr, + const unsigned start_len, const unsigned max_len) +{ + unsigned len = start_len; + machine_word_t v_word; + + if (UNALIGNED_ACCESS_IS_FAST) { + + if (likely(max_len - len >= 4 * WORDBYTES)) { + + #define COMPARE_WORD_STEP \ + v_word = load_word_unaligned(&matchptr[len]) ^ \ + load_word_unaligned(&strptr[len]); \ + if (v_word != 0) \ + goto word_differs; \ + len += WORDBYTES; \ + + COMPARE_WORD_STEP + COMPARE_WORD_STEP + COMPARE_WORD_STEP + COMPARE_WORD_STEP + #undef COMPARE_WORD_STEP + } + + while (len + WORDBYTES <= max_len) { + v_word = load_word_unaligned(&matchptr[len]) ^ + load_word_unaligned(&strptr[len]); + if (v_word != 0) + goto word_differs; + len += WORDBYTES; + } + } + + while (len < max_len && matchptr[len] == strptr[len]) + len++; + return len; + +word_differs: + if (CPU_IS_LITTLE_ENDIAN()) + len += (bsfw(v_word) >> 3); + else + len += (WORDBITS - 1 - bsrw(v_word)) >> 3; + return len; +} + +#endif /* LIB_MATCHFINDER_COMMON_H */ diff --git a/ext/libdeflate/lib/unaligned.h b/ext/libdeflate/lib/unaligned.h new file mode 100644 index 0000000000..bb48bf8283 --- /dev/null +++ b/ext/libdeflate/lib/unaligned.h @@ -0,0 +1,228 @@ +/* + * unaligned.h - inline functions for unaligned memory accesses + */ + +#ifndef LIB_UNALIGNED_H +#define LIB_UNALIGNED_H + +#include "lib_common.h" + +/***** Unaligned loads and stores without endianness conversion *****/ + +/* + * memcpy() is portable, and it usually gets optimized appropriately by modern + * compilers. I.e., each memcpy() of 1, 2, 4, or WORDBYTES bytes gets compiled + * to a load or store instruction, not to an actual function call. + * + * We no longer use the "packed struct" approach, as that is nonstandard, has + * unclear semantics, and doesn't receive enough testing + * (see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=94994). + * + * arm32 with __ARM_FEATURE_UNALIGNED in gcc 5 and earlier is a known exception + * where memcpy() generates inefficient code + * (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67366). However, we no longer + * consider that one case important enough to maintain different code for. + * If you run into it, please just use a newer version of gcc (or use clang). + */ + +#define DEFINE_UNALIGNED_TYPE(type) \ +static forceinline type \ +load_##type##_unaligned(const void *p) \ +{ \ + type v; \ + memcpy(&v, p, sizeof(v)); \ + return v; \ +} \ + \ +static forceinline void \ +store_##type##_unaligned(type v, void *p) \ +{ \ + memcpy(p, &v, sizeof(v)); \ +} + +DEFINE_UNALIGNED_TYPE(u16) +DEFINE_UNALIGNED_TYPE(u32) +DEFINE_UNALIGNED_TYPE(u64) +DEFINE_UNALIGNED_TYPE(machine_word_t) + +#define load_word_unaligned load_machine_word_t_unaligned +#define store_word_unaligned store_machine_word_t_unaligned + +/***** Unaligned loads with endianness conversion *****/ + +static forceinline u16 +get_unaligned_le16(const u8 *p) +{ + if (UNALIGNED_ACCESS_IS_FAST) + return le16_bswap(load_u16_unaligned(p)); + else + return ((u16)p[1] << 8) | p[0]; +} + +static forceinline u16 +get_unaligned_be16(const u8 *p) +{ + if (UNALIGNED_ACCESS_IS_FAST) + return be16_bswap(load_u16_unaligned(p)); + else + return ((u16)p[0] << 8) | p[1]; +} + +static forceinline u32 +get_unaligned_le32(const u8 *p) +{ + if (UNALIGNED_ACCESS_IS_FAST) + return le32_bswap(load_u32_unaligned(p)); + else + return ((u32)p[3] << 24) | ((u32)p[2] << 16) | + ((u32)p[1] << 8) | p[0]; +} + +static forceinline u32 +get_unaligned_be32(const u8 *p) +{ + if (UNALIGNED_ACCESS_IS_FAST) + return be32_bswap(load_u32_unaligned(p)); + else + return ((u32)p[0] << 24) | ((u32)p[1] << 16) | + ((u32)p[2] << 8) | p[3]; +} + +static forceinline u64 +get_unaligned_le64(const u8 *p) +{ + if (UNALIGNED_ACCESS_IS_FAST) + return le64_bswap(load_u64_unaligned(p)); + else + return ((u64)p[7] << 56) | ((u64)p[6] << 48) | + ((u64)p[5] << 40) | ((u64)p[4] << 32) | + ((u64)p[3] << 24) | ((u64)p[2] << 16) | + ((u64)p[1] << 8) | p[0]; +} + +static forceinline machine_word_t +get_unaligned_leword(const u8 *p) +{ + STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64); + if (WORDBITS == 32) + return get_unaligned_le32(p); + else + return get_unaligned_le64(p); +} + +/***** Unaligned stores with endianness conversion *****/ + +static forceinline void +put_unaligned_le16(u16 v, u8 *p) +{ + if (UNALIGNED_ACCESS_IS_FAST) { + store_u16_unaligned(le16_bswap(v), p); + } else { + p[0] = (u8)(v >> 0); + p[1] = (u8)(v >> 8); + } +} + +static forceinline void +put_unaligned_be16(u16 v, u8 *p) +{ + if (UNALIGNED_ACCESS_IS_FAST) { + store_u16_unaligned(be16_bswap(v), p); + } else { + p[0] = (u8)(v >> 8); + p[1] = (u8)(v >> 0); + } +} + +static forceinline void +put_unaligned_le32(u32 v, u8 *p) +{ + if (UNALIGNED_ACCESS_IS_FAST) { + store_u32_unaligned(le32_bswap(v), p); + } else { + p[0] = (u8)(v >> 0); + p[1] = (u8)(v >> 8); + p[2] = (u8)(v >> 16); + p[3] = (u8)(v >> 24); + } +} + +static forceinline void +put_unaligned_be32(u32 v, u8 *p) +{ + if (UNALIGNED_ACCESS_IS_FAST) { + store_u32_unaligned(be32_bswap(v), p); + } else { + p[0] = (u8)(v >> 24); + p[1] = (u8)(v >> 16); + p[2] = (u8)(v >> 8); + p[3] = (u8)(v >> 0); + } +} + +static forceinline void +put_unaligned_le64(u64 v, u8 *p) +{ + if (UNALIGNED_ACCESS_IS_FAST) { + store_u64_unaligned(le64_bswap(v), p); + } else { + p[0] = (u8)(v >> 0); + p[1] = (u8)(v >> 8); + p[2] = (u8)(v >> 16); + p[3] = (u8)(v >> 24); + p[4] = (u8)(v >> 32); + p[5] = (u8)(v >> 40); + p[6] = (u8)(v >> 48); + p[7] = (u8)(v >> 56); + } +} + +static forceinline void +put_unaligned_leword(machine_word_t v, u8 *p) +{ + STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64); + if (WORDBITS == 32) + put_unaligned_le32(v, p); + else + put_unaligned_le64(v, p); +} + +/***** 24-bit loads *****/ + +/* + * Given a 32-bit value that was loaded with the platform's native endianness, + * return a 32-bit value whose high-order 8 bits are 0 and whose low-order 24 + * bits contain the first 3 bytes, arranged in octets in a platform-dependent + * order, at the memory location from which the input 32-bit value was loaded. + */ +static forceinline u32 +loaded_u32_to_u24(u32 v) +{ + if (CPU_IS_LITTLE_ENDIAN()) + return v & 0xFFFFFF; + else + return v >> 8; +} + +/* + * Load the next 3 bytes from the memory location @p into the 24 low-order bits + * of a 32-bit value. The order in which the 3 bytes will be arranged as octets + * in the 24 bits is platform-dependent. At least LOAD_U24_REQUIRED_NBYTES + * bytes must be available at @p; note that this may be more than 3. + */ +static forceinline u32 +load_u24_unaligned(const u8 *p) +{ +#if UNALIGNED_ACCESS_IS_FAST +# define LOAD_U24_REQUIRED_NBYTES 4 + return loaded_u32_to_u24(load_u32_unaligned(p)); +#else +# define LOAD_U24_REQUIRED_NBYTES 3 + if (CPU_IS_LITTLE_ENDIAN()) + return ((u32)p[0] << 0) | ((u32)p[1] << 8) | ((u32)p[2] << 16); + else + return ((u32)p[2] << 0) | ((u32)p[1] << 8) | ((u32)p[0] << 16); +#endif +} + +#endif /* LIB_UNALIGNED_H */ diff --git a/ext/libdeflate/lib/utils.c b/ext/libdeflate/lib/utils.c new file mode 100644 index 0000000000..ed76046368 --- /dev/null +++ b/ext/libdeflate/lib/utils.c @@ -0,0 +1,138 @@ +/* + * utils.c - utility functions for libdeflate + * + * Copyright 2016 Eric Biggers + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "lib_common.h" + +#include "../libdeflate.h" + +#ifdef FREESTANDING +# define malloc NULL +# define free NULL +#else +# include +#endif + +static void *(*libdeflate_malloc_func)(size_t) = malloc; +static void (*libdeflate_free_func)(void *) = free; + +void * +libdeflate_malloc(size_t size) +{ + return (*libdeflate_malloc_func)(size); +} + +void +libdeflate_free(void *ptr) +{ + (*libdeflate_free_func)(ptr); +} + +void * +libdeflate_aligned_malloc(size_t alignment, size_t size) +{ + void *ptr = libdeflate_malloc(sizeof(void *) + alignment - 1 + size); + if (ptr) { + void *orig_ptr = ptr; + ptr = (void *)ALIGN((uintptr_t)ptr + sizeof(void *), alignment); + ((void **)ptr)[-1] = orig_ptr; + } + return ptr; +} + +void +libdeflate_aligned_free(void *ptr) +{ + if (ptr) + libdeflate_free(((void **)ptr)[-1]); +} + +LIBDEFLATEEXPORT void LIBDEFLATEAPI +libdeflate_set_memory_allocator(void *(*malloc_func)(size_t), + void (*free_func)(void *)) +{ + libdeflate_malloc_func = malloc_func; + libdeflate_free_func = free_func; +} + +/* + * Implementations of libc functions for freestanding library builds. + * Normal library builds don't use these. Not optimized yet; usually the + * compiler expands these functions and doesn't actually call them anyway. + */ +#ifdef FREESTANDING +#undef memset +void *memset(void *s, int c, size_t n) +{ + u8 *p = s; + size_t i; + + for (i = 0; i < n; i++) + p[i] = c; + return s; +} + +#undef memcpy +void *memcpy(void *dest, const void *src, size_t n) +{ + u8 *d = dest; + const u8 *s = src; + size_t i; + + for (i = 0; i < n; i++) + d[i] = s[i]; + return dest; +} + +#undef memmove +void *memmove(void *dest, const void *src, size_t n) +{ + u8 *d = dest; + const u8 *s = src; + size_t i; + + if (d <= s) + return memcpy(d, s, n); + + for (i = n; i > 0; i--) + d[i - 1] = s[i - 1]; + return dest; +} + +#undef memcmp +int memcmp(const void *s1, const void *s2, size_t n) +{ + const u8 *p1 = s1; + const u8 *p2 = s2; + size_t i; + + for (i = 0; i < n; i++) { + if (p1[i] != p2[i]) + return (int)p1[i] - (int)p2[i]; + } + return 0; +} +#endif /* FREESTANDING */ diff --git a/ext/libdeflate/lib/x86/matchfinder_impl.h b/ext/libdeflate/lib/x86/matchfinder_impl.h new file mode 100644 index 0000000000..22a6c82da3 --- /dev/null +++ b/ext/libdeflate/lib/x86/matchfinder_impl.h @@ -0,0 +1,117 @@ +/* + * x86/matchfinder_impl.h - x86 implementations of matchfinder functions + * + * Copyright 2016 Eric Biggers + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifdef __AVX2__ +# include +static forceinline void +matchfinder_init_avx2(mf_pos_t *data, size_t size) +{ + __m256i *p = (__m256i *)data; + __m256i v = _mm256_set1_epi16(MATCHFINDER_INITVAL); + + STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0); + STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0); + STATIC_ASSERT(sizeof(mf_pos_t) == 2); + + do { + p[0] = v; + p[1] = v; + p[2] = v; + p[3] = v; + p += 4; + size -= 4 * sizeof(*p); + } while (size != 0); +} +#define matchfinder_init matchfinder_init_avx2 + +static forceinline void +matchfinder_rebase_avx2(mf_pos_t *data, size_t size) +{ + __m256i *p = (__m256i *)data; + __m256i v = _mm256_set1_epi16((u16)-MATCHFINDER_WINDOW_SIZE); + + STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0); + STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0); + STATIC_ASSERT(sizeof(mf_pos_t) == 2); + + do { + /* PADDSW: Add Packed Signed Integers With Signed Saturation */ + p[0] = _mm256_adds_epi16(p[0], v); + p[1] = _mm256_adds_epi16(p[1], v); + p[2] = _mm256_adds_epi16(p[2], v); + p[3] = _mm256_adds_epi16(p[3], v); + p += 4; + size -= 4 * sizeof(*p); + } while (size != 0); +} +#define matchfinder_rebase matchfinder_rebase_avx2 + +#elif defined(__SSE2__) +# include +static forceinline void +matchfinder_init_sse2(mf_pos_t *data, size_t size) +{ + __m128i *p = (__m128i *)data; + __m128i v = _mm_set1_epi16(MATCHFINDER_INITVAL); + + STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0); + STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0); + STATIC_ASSERT(sizeof(mf_pos_t) == 2); + + do { + p[0] = v; + p[1] = v; + p[2] = v; + p[3] = v; + p += 4; + size -= 4 * sizeof(*p); + } while (size != 0); +} +#define matchfinder_init matchfinder_init_sse2 + +static forceinline void +matchfinder_rebase_sse2(mf_pos_t *data, size_t size) +{ + __m128i *p = (__m128i *)data; + __m128i v = _mm_set1_epi16((u16)-MATCHFINDER_WINDOW_SIZE); + + STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0); + STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0); + STATIC_ASSERT(sizeof(mf_pos_t) == 2); + + do { + /* PADDSW: Add Packed Signed Integers With Signed Saturation */ + p[0] = _mm_adds_epi16(p[0], v); + p[1] = _mm_adds_epi16(p[1], v); + p[2] = _mm_adds_epi16(p[2], v); + p[3] = _mm_adds_epi16(p[3], v); + p += 4; + size -= 4 * sizeof(*p); + } while (size != 0); +} +#define matchfinder_rebase matchfinder_rebase_sse2 +#endif /* __SSE2__ */ diff --git a/ext/libdeflate/libdeflate.h b/ext/libdeflate/libdeflate.h new file mode 100644 index 0000000000..cbcda04202 --- /dev/null +++ b/ext/libdeflate/libdeflate.h @@ -0,0 +1,366 @@ +/* + * libdeflate.h - public header for libdeflate + */ + +#ifndef LIBDEFLATE_H +#define LIBDEFLATE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define LIBDEFLATE_VERSION_MAJOR 1 +#define LIBDEFLATE_VERSION_MINOR 7 +#define LIBDEFLATE_VERSION_STRING "1.7" + +#include +#include + +/* + * On Windows, if you want to link to the DLL version of libdeflate, then + * #define LIBDEFLATE_DLL. Note that the calling convention is "stdcall". + */ +#ifdef LIBDEFLATE_DLL +# ifdef BUILDING_LIBDEFLATE +# define LIBDEFLATEEXPORT LIBEXPORT +# elif defined(_WIN32) || defined(__CYGWIN__) +# define LIBDEFLATEEXPORT __declspec(dllimport) +# endif +#endif +#ifndef LIBDEFLATEEXPORT +# define LIBDEFLATEEXPORT +#endif + +#if defined(_WIN32) && !defined(_WIN64) +# define LIBDEFLATEAPI_ABI __stdcall +#else +# define LIBDEFLATEAPI_ABI +#endif + +#if defined(BUILDING_LIBDEFLATE) && defined(__GNUC__) && \ + defined(_WIN32) && !defined(_WIN64) + /* + * On 32-bit Windows, gcc assumes 16-byte stack alignment but MSVC only 4. + * Realign the stack when entering libdeflate to avoid crashing in SSE/AVX + * code when called from an MSVC-compiled application. + */ +# define LIBDEFLATEAPI_STACKALIGN __attribute__((force_align_arg_pointer)) +#else +# define LIBDEFLATEAPI_STACKALIGN +#endif + +#define LIBDEFLATEAPI LIBDEFLATEAPI_ABI LIBDEFLATEAPI_STACKALIGN + +/* ========================================================================== */ +/* Compression */ +/* ========================================================================== */ + +struct libdeflate_compressor; + +/* + * libdeflate_alloc_compressor() allocates a new compressor that supports + * DEFLATE, zlib, and gzip compression. 'compression_level' is the compression + * level on a zlib-like scale but with a higher maximum value (1 = fastest, 6 = + * medium/default, 9 = slow, 12 = slowest). Level 0 is also supported and means + * "no compression", specifically "create a valid stream, but only emit + * uncompressed blocks" (this will expand the data slightly). + * + * The return value is a pointer to the new compressor, or NULL if out of memory + * or if the compression level is invalid (i.e. outside the range [0, 12]). + * + * Note: for compression, the sliding window size is defined at compilation time + * to 32768, the largest size permissible in the DEFLATE format. It cannot be + * changed at runtime. + * + * A single compressor is not safe to use by multiple threads concurrently. + * However, different threads may use different compressors concurrently. + */ +LIBDEFLATEEXPORT struct libdeflate_compressor * LIBDEFLATEAPI +libdeflate_alloc_compressor(int compression_level); + +/* + * libdeflate_deflate_compress() performs raw DEFLATE compression on a buffer of + * data. The function attempts to compress 'in_nbytes' bytes of data located at + * 'in' and write the results to 'out', which has space for 'out_nbytes_avail' + * bytes. The return value is the compressed size in bytes, or 0 if the data + * could not be compressed to 'out_nbytes_avail' bytes or fewer. + */ +LIBDEFLATEEXPORT size_t LIBDEFLATEAPI +libdeflate_deflate_compress(struct libdeflate_compressor *compressor, + const void *in, size_t in_nbytes, + void *out, size_t out_nbytes_avail); + +/* + * libdeflate_deflate_compress_bound() returns a worst-case upper bound on the + * number of bytes of compressed data that may be produced by compressing any + * buffer of length less than or equal to 'in_nbytes' using + * libdeflate_deflate_compress() with the specified compressor. Mathematically, + * this bound will necessarily be a number greater than or equal to 'in_nbytes'. + * It may be an overestimate of the true upper bound. The return value is + * guaranteed to be the same for all invocations with the same compressor and + * same 'in_nbytes'. + * + * As a special case, 'compressor' may be NULL. This causes the bound to be + * taken across *any* libdeflate_compressor that could ever be allocated with + * this build of the library, with any options. + * + * Note that this function is not necessary in many applications. With + * block-based compression, it is usually preferable to separately store the + * uncompressed size of each block and to store any blocks that did not compress + * to less than their original size uncompressed. In that scenario, there is no + * need to know the worst-case compressed size, since the maximum number of + * bytes of compressed data that may be used would always be one less than the + * input length. You can just pass a buffer of that size to + * libdeflate_deflate_compress() and store the data uncompressed if + * libdeflate_deflate_compress() returns 0, indicating that the compressed data + * did not fit into the provided output buffer. + */ +LIBDEFLATEEXPORT size_t LIBDEFLATEAPI +libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor, + size_t in_nbytes); + +/* + * Like libdeflate_deflate_compress(), but stores the data in the zlib wrapper + * format. + */ +LIBDEFLATEEXPORT size_t LIBDEFLATEAPI +libdeflate_zlib_compress(struct libdeflate_compressor *compressor, + const void *in, size_t in_nbytes, + void *out, size_t out_nbytes_avail); + +/* + * Like libdeflate_deflate_compress_bound(), but assumes the data will be + * compressed with libdeflate_zlib_compress() rather than with + * libdeflate_deflate_compress(). + */ +LIBDEFLATEEXPORT size_t LIBDEFLATEAPI +libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor, + size_t in_nbytes); + +/* + * Like libdeflate_deflate_compress(), but stores the data in the gzip wrapper + * format. + */ +LIBDEFLATEEXPORT size_t LIBDEFLATEAPI +libdeflate_gzip_compress(struct libdeflate_compressor *compressor, + const void *in, size_t in_nbytes, + void *out, size_t out_nbytes_avail); + +/* + * Like libdeflate_deflate_compress_bound(), but assumes the data will be + * compressed with libdeflate_gzip_compress() rather than with + * libdeflate_deflate_compress(). + */ +LIBDEFLATEEXPORT size_t LIBDEFLATEAPI +libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor, + size_t in_nbytes); + +/* + * libdeflate_free_compressor() frees a compressor that was allocated with + * libdeflate_alloc_compressor(). If a NULL pointer is passed in, no action is + * taken. + */ +LIBDEFLATEEXPORT void LIBDEFLATEAPI +libdeflate_free_compressor(struct libdeflate_compressor *compressor); + +/* ========================================================================== */ +/* Decompression */ +/* ========================================================================== */ + +struct libdeflate_decompressor; + +/* + * libdeflate_alloc_decompressor() allocates a new decompressor that can be used + * for DEFLATE, zlib, and gzip decompression. The return value is a pointer to + * the new decompressor, or NULL if out of memory. + * + * This function takes no parameters, and the returned decompressor is valid for + * decompressing data that was compressed at any compression level and with any + * sliding window size. + * + * A single decompressor is not safe to use by multiple threads concurrently. + * However, different threads may use different decompressors concurrently. + */ +LIBDEFLATEEXPORT struct libdeflate_decompressor * LIBDEFLATEAPI +libdeflate_alloc_decompressor(void); + +/* + * Result of a call to libdeflate_deflate_decompress(), + * libdeflate_zlib_decompress(), or libdeflate_gzip_decompress(). + */ +enum libdeflate_result { + /* Decompression was successful. */ + LIBDEFLATE_SUCCESS = 0, + + /* Decompressed failed because the compressed data was invalid, corrupt, + * or otherwise unsupported. */ + LIBDEFLATE_BAD_DATA = 1, + + /* A NULL 'actual_out_nbytes_ret' was provided, but the data would have + * decompressed to fewer than 'out_nbytes_avail' bytes. */ + LIBDEFLATE_SHORT_OUTPUT = 2, + + /* The data would have decompressed to more than 'out_nbytes_avail' + * bytes. */ + LIBDEFLATE_INSUFFICIENT_SPACE = 3, +}; + +/* + * libdeflate_deflate_decompress() decompresses the DEFLATE-compressed stream + * from the buffer 'in' with compressed size up to 'in_nbytes' bytes. The + * uncompressed data is written to 'out', a buffer with size 'out_nbytes_avail' + * bytes. If decompression succeeds, then 0 (LIBDEFLATE_SUCCESS) is returned. + * Otherwise, a nonzero result code such as LIBDEFLATE_BAD_DATA is returned. If + * a nonzero result code is returned, then the contents of the output buffer are + * undefined. + * + * Decompression stops at the end of the DEFLATE stream (as indicated by the + * BFINAL flag), even if it is actually shorter than 'in_nbytes' bytes. + * + * libdeflate_deflate_decompress() can be used in cases where the actual + * uncompressed size is known (recommended) or unknown (not recommended): + * + * - If the actual uncompressed size is known, then pass the actual + * uncompressed size as 'out_nbytes_avail' and pass NULL for + * 'actual_out_nbytes_ret'. This makes libdeflate_deflate_decompress() fail + * with LIBDEFLATE_SHORT_OUTPUT if the data decompressed to fewer than the + * specified number of bytes. + * + * - If the actual uncompressed size is unknown, then provide a non-NULL + * 'actual_out_nbytes_ret' and provide a buffer with some size + * 'out_nbytes_avail' that you think is large enough to hold all the + * uncompressed data. In this case, if the data decompresses to less than + * or equal to 'out_nbytes_avail' bytes, then + * libdeflate_deflate_decompress() will write the actual uncompressed size + * to *actual_out_nbytes_ret and return 0 (LIBDEFLATE_SUCCESS). Otherwise, + * it will return LIBDEFLATE_INSUFFICIENT_SPACE if the provided buffer was + * not large enough but no other problems were encountered, or another + * nonzero result code if decompression failed for another reason. + */ +LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI +libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor, + const void *in, size_t in_nbytes, + void *out, size_t out_nbytes_avail, + size_t *actual_out_nbytes_ret); + +/* + * Like libdeflate_deflate_decompress(), but adds the 'actual_in_nbytes_ret' + * argument. If decompression succeeds and 'actual_in_nbytes_ret' is not NULL, + * then the actual compressed size of the DEFLATE stream (aligned to the next + * byte boundary) is written to *actual_in_nbytes_ret. + */ +LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI +libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor, + const void *in, size_t in_nbytes, + void *out, size_t out_nbytes_avail, + size_t *actual_in_nbytes_ret, + size_t *actual_out_nbytes_ret); + +/* + * Like libdeflate_deflate_decompress(), but assumes the zlib wrapper format + * instead of raw DEFLATE. + * + * Decompression will stop at the end of the zlib stream, even if it is shorter + * than 'in_nbytes'. If you need to know exactly where the zlib stream ended, + * use libdeflate_zlib_decompress_ex(). + */ +LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI +libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor, + const void *in, size_t in_nbytes, + void *out, size_t out_nbytes_avail, + size_t *actual_out_nbytes_ret); + +/* + * Like libdeflate_zlib_decompress(), but adds the 'actual_in_nbytes_ret' + * argument. If 'actual_in_nbytes_ret' is not NULL and the decompression + * succeeds (indicating that the first zlib-compressed stream in the input + * buffer was decompressed), then the actual number of input bytes consumed is + * written to *actual_in_nbytes_ret. + */ +LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI +libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor, + const void *in, size_t in_nbytes, + void *out, size_t out_nbytes_avail, + size_t *actual_in_nbytes_ret, + size_t *actual_out_nbytes_ret); + +/* + * Like libdeflate_deflate_decompress(), but assumes the gzip wrapper format + * instead of raw DEFLATE. + * + * If multiple gzip-compressed members are concatenated, then only the first + * will be decompressed. Use libdeflate_gzip_decompress_ex() if you need + * multi-member support. + */ +LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI +libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor, + const void *in, size_t in_nbytes, + void *out, size_t out_nbytes_avail, + size_t *actual_out_nbytes_ret); + +/* + * Like libdeflate_gzip_decompress(), but adds the 'actual_in_nbytes_ret' + * argument. If 'actual_in_nbytes_ret' is not NULL and the decompression + * succeeds (indicating that the first gzip-compressed member in the input + * buffer was decompressed), then the actual number of input bytes consumed is + * written to *actual_in_nbytes_ret. + */ +LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI +libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor, + const void *in, size_t in_nbytes, + void *out, size_t out_nbytes_avail, + size_t *actual_in_nbytes_ret, + size_t *actual_out_nbytes_ret); + +/* + * libdeflate_free_decompressor() frees a decompressor that was allocated with + * libdeflate_alloc_decompressor(). If a NULL pointer is passed in, no action + * is taken. + */ +LIBDEFLATEEXPORT void LIBDEFLATEAPI +libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor); + +/* ========================================================================== */ +/* Checksums */ +/* ========================================================================== */ + +/* + * libdeflate_adler32() updates a running Adler-32 checksum with 'len' bytes of + * data and returns the updated checksum. When starting a new checksum, the + * required initial value for 'adler' is 1. This value is also returned when + * 'buffer' is specified as NULL. + */ +LIBDEFLATEEXPORT uint32_t LIBDEFLATEAPI +libdeflate_adler32(uint32_t adler, const void *buffer, size_t len); + + +/* + * libdeflate_crc32() updates a running CRC-32 checksum with 'len' bytes of data + * and returns the updated checksum. When starting a new checksum, the required + * initial value for 'crc' is 0. This value is also returned when 'buffer' is + * specified as NULL. + */ +LIBDEFLATEEXPORT uint32_t LIBDEFLATEAPI +libdeflate_crc32(uint32_t crc, const void *buffer, size_t len); + +/* ========================================================================== */ +/* Custom memory allocator */ +/* ========================================================================== */ + +/* + * Install a custom memory allocator which libdeflate will use for all memory + * allocations. 'malloc_func' is a function that must behave like malloc(), and + * 'free_func' is a function that must behave like free(). + * + * There must not be any libdeflate_compressor or libdeflate_decompressor + * structures in existence when calling this function. + */ +LIBDEFLATEEXPORT void LIBDEFLATEAPI +libdeflate_set_memory_allocator(void *(*malloc_func)(size_t), + void (*free_func)(void *)); + +#ifdef __cplusplus +} +#endif + +#endif /* LIBDEFLATE_H */ diff --git a/libretro/Makefile.common b/libretro/Makefile.common index 7dab17fab4..a712a07cd6 100644 --- a/libretro/Makefile.common +++ b/libretro/Makefile.common @@ -398,6 +398,8 @@ SOURCES_CXX += \ SOURCES_C += $(EXTDIR)/sfmt19937/SFMT.c SOURCES_C += $(EXTDIR)/xxhash.c +SOURCES_C += $(EXTDIR)/libdeflate/lib/deflate_compress.c \ + $(EXTDIR)/libdeflate/lib/utils.c SOURCES_CXX += \ $(EXTDIR)/snappy/snappy-c.cpp \ From 363d8167899025dbd681093711794d37c1ef9bd4 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 16:01:53 +0200 Subject: [PATCH 2/2] Game screen: Compress game as CSO "Compress game" in the game screen's ... menu, for ISO, CSO, CHD and PSN EBOOT.PBP games. It opens a list of the CSO presets (and Extract to ISO for anything that's not an ISO already): - Best: libdeflate level 12, 64K blocks - PSP Compatible: zlib level 9, 2K blocks, readable by custom firmware These match maxcso's presets, and the CSOs come out byte-identical to its. The writer, Core/FileSystems/CSOCompress, reads the input through PPSSPP's own block devices, so recompressing is a single pass with no temporary file. The original is deleted when done, unless "Keep the original file" is checked (off by default). When the new name is taken, by a kept original or another file, a number is added to it. The work runs on a thread using every core, with a progress bar and time left at the bottom of the game screen. It keeps going when the screen is left, and the ... menu offers Cancel meanwhile. On the game list the game's icon is covered by "Compressing...", and clicking it opens the game screen instead of playing. When done, recents follow the new file and the game screen and lists refresh. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CCxekYqPnzNr5d8Tp1q7aZ --- Core/CMakeLists.txt | 2 + Core/Config.cpp | 2 + Core/Config.h | 3 + Core/Core.vcxproj | 2 + Core/Core.vcxproj.filters | 6 + Core/FileSystems/CSOCompress.cpp | 354 ++++++++++++++++++++ Core/FileSystems/CSOCompress.h | 43 +++ UI/CMakeLists.txt | 2 + UI/GameBrowser.cpp | 15 + UI/GamePacker.cpp | 478 ++++++++++++++++++++++++++++ UI/GamePacker.h | 109 +++++++ UI/GameScreen.cpp | 100 +++++- UI/GameScreen.h | 6 + UI/MainScreen.cpp | 20 ++ UI/MainScreen.h | 1 + UI/NativeApp.cpp | 6 + UI/UI.vcxproj | 2 + UI/UI.vcxproj.filters | 6 + UWP/CoreUWP/CoreUWP.vcxproj | 2 + UWP/CoreUWP/CoreUWP.vcxproj.filters | 2 + UWP/UI_UWP/UI_UWP.vcxproj | 2 + UWP/UI_UWP/UI_UWP.vcxproj.filters | 6 + android/jni/Android.mk | 2 + libretro/Makefile.common | 1 + 24 files changed, 1166 insertions(+), 6 deletions(-) create mode 100644 Core/FileSystems/CSOCompress.cpp create mode 100644 Core/FileSystems/CSOCompress.h create mode 100644 UI/GamePacker.cpp create mode 100644 UI/GamePacker.h diff --git a/Core/CMakeLists.txt b/Core/CMakeLists.txt index 15bc787467..1c9a19417f 100644 --- a/Core/CMakeLists.txt +++ b/Core/CMakeLists.txt @@ -357,6 +357,8 @@ add_library(Core STATIC FileSystems/BlobFileSystem.h FileSystems/BlockDevices.cpp FileSystems/BlockDevices.h + FileSystems/CSOCompress.cpp + FileSystems/CSOCompress.h FileSystems/DirectoryFileSystem.cpp FileSystems/DirectoryFileSystem.h FileSystems/FileSystem.h diff --git a/Core/Config.cpp b/Core/Config.cpp index a15dc9ea1f..c12992fe0a 100644 --- a/Core/Config.cpp +++ b/Core/Config.cpp @@ -328,6 +328,8 @@ static const ConfigSetting generalSettings[] = { ConfigSetting("CompressSymbols", SETTING(g_Config, bCompressSymbols), true, CfgFlag::DEFAULT), ConfigSetting("CacheFullIsoInRam", SETTING(g_Config, bCacheFullIsoInRam), false, CfgFlag::PER_GAME), ConfigSetting("AutoUpgradeFirmware", SETTING(g_Config, bAutoUpgradeFirmware), true, CfgFlag::DEFAULT), + ConfigSetting("CSOPreset", SETTING(g_Config, iCSOPreset), 0, CfgFlag::DEFAULT), + ConfigSetting("PackKeepOriginal", SETTING(g_Config, bPackKeepOriginal), false, CfgFlag::DEFAULT), ConfigSetting("RemoteISOPort", SETTING(g_Config, iRemoteISOPort), 0, CfgFlag::DEFAULT), ConfigSetting("LastRemoteISOServer", SETTING(g_Config, sLastRemoteISOServer), "", CfgFlag::DEFAULT), ConfigSetting("LastRemoteISOPort", SETTING(g_Config, iLastRemoteISOPort), 0, CfgFlag::DEFAULT), diff --git a/Core/Config.h b/Core/Config.h index 3adb876415..8e75c10fa0 100644 --- a/Core/Config.h +++ b/Core/Config.h @@ -253,6 +253,9 @@ public: bool bCompressSymbols; bool bCacheFullIsoInRam; bool bAutoUpgradeFirmware; + // The game screen's "Compress game" (UI/GamePacker.h). + int iCSOPreset; // CSOCompress::Preset: 0 = Best, 1 = PSP Compatible + bool bPackKeepOriginal; int iRemoteISOPort; // Also used for serving a local remote debugger. std::string sLastRemoteISOServer; int iLastRemoteISOPort; diff --git a/Core/Core.vcxproj b/Core/Core.vcxproj index 02835fcd16..c413dd7ed7 100644 --- a/Core/Core.vcxproj +++ b/Core/Core.vcxproj @@ -585,6 +585,7 @@ + @@ -921,6 +922,7 @@ + diff --git a/Core/Core.vcxproj.filters b/Core/Core.vcxproj.filters index bb4a8a3126..1ec7c1115c 100644 --- a/Core/Core.vcxproj.filters +++ b/Core/Core.vcxproj.filters @@ -144,6 +144,9 @@ FileSystems + + FileSystems + FileSystems @@ -1431,6 +1434,9 @@ FileSystems + + FileSystems + FileSystems diff --git a/Core/FileSystems/CSOCompress.cpp b/Core/FileSystems/CSOCompress.cpp new file mode 100644 index 0000000000..1337efd5b2 --- /dev/null +++ b/Core/FileSystems/CSOCompress.cpp @@ -0,0 +1,354 @@ +// Copyright (c) 2012- PPSSPP Project. + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, version 2.0 or later versions. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License 2.0 for more details. + +// A copy of the GPL 2.0 should have been included with the program. +// If not, see http://www.gnu.org/licenses/ + +// Official git repository and contact information can be found at +// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Common/CommonTypes.h" +#include "Common/File/FileUtil.h" +#include "Common/Log.h" +#include "Common/Swap.h" +#include "Common/Thread/ThreadUtil.h" +#include "Core/FileSystems/BlockDevices.h" +#include "Core/Loaders.h" +#include "Core/FileSystems/CSOCompress.h" + +#ifdef SHARED_ZLIB +#include +#else +#include "ext/zlib/zlib.h" +#endif + +#include "ext/libdeflate/libdeflate.h" + +namespace CSOCompress { + +namespace { + +const uint32_t SECTOR_SIZE = 2048; +const uint32_t CSO_INDEX_UNCOMPRESSED = 0x80000000; +// Input read and compressed per round, so the threads always have work queued. +const size_t BATCH_BYTES = 32 * 1024 * 1024; + +struct PresetSettings { + uint32_t blockSize; + bool zlib; // zlib level 9, default strategy. + bool libdeflate; // libdeflate level 12. +}; + +// The trials maxcso's presets ran (with lz4, 7-zip and zlib's brute force strategies left out). +PresetSettings SettingsFor(Preset preset) { + switch (preset) { + case Preset::Best: + // 64K: only ~0.25% bigger than 256K, and each read of a new block inflates a quarter as much. + return { 65536, false, true }; + case Preset::PSPCompatible: + default: + // Some custom firmware can't read libdeflate's output or larger blocks. + return { 2048, true, false }; + } +} + +// One thread's encoders, reused for every block it compresses. +class BlockCompressor { +public: + explicit BlockCompressor(const PresetSettings &settings) : settings_(settings), scratch_(settings.blockSize) { + if (settings_.zlib) { + zlibOk_ = deflateInit2(&z_, 9, Z_DEFLATED, -15, 9, Z_DEFAULT_STRATEGY) == Z_OK; + } + if (settings_.libdeflate) { + libdeflate_ = libdeflate_alloc_compressor(12); + } + } + ~BlockCompressor() { + if (zlibOk_) { + deflateEnd(&z_); + } + if (libdeflate_) { + libdeflate_free_compressor(libdeflate_); + } + } + + // Leaves the smallest raw deflate stream in *out, or nothing when none beats the block itself. + void Compress(const u8 *in, std::vector *out) { + const uint32_t blockSize = settings_.blockSize; + out->clear(); + auto submit = [&](const u8 *data, size_t size) { + // Not smaller than the block: stored as is, which also reads faster. + if (size > 0 && size < blockSize && (out->empty() || size < out->size())) { + out->assign(data, data + size); + } + }; + + if (libdeflate_) { + // libdeflate returns 0 when the result doesn't fit, i.e. isn't smaller. + const size_t size = libdeflate_deflate_compress(libdeflate_, in, blockSize, scratch_.data(), scratch_.size()); + submit(scratch_.data(), size); + } + if (zlibOk_ && deflateReset(&z_) == Z_OK) { + z_.next_in = (Bytef *)in; + z_.avail_in = blockSize; + z_.next_out = scratch_.data(); + z_.avail_out = (uInt)scratch_.size(); + int res; + while ((res = deflate(&z_, Z_FINISH)) == Z_OK) { + continue; + } + if (res == Z_STREAM_END) { + submit(scratch_.data(), z_.total_out); + } + } + } + +private: + PresetSettings settings_; + std::vector scratch_; + z_stream z_{}; + bool zlibOk_ = false; + libdeflate_compressor *libdeflate_ = nullptr; +}; + +// Opens anything PPSSPP plays as a disc, as 2048-byte sectors. +struct Source { + std::unique_ptr loader; + std::unique_ptr device; + uint64_t size = 0; + + bool Open(const Path &path, std::string *error) { + loader.reset(ConstructFileLoader(path)); + if (!loader || !loader->Exists()) { + *error = "Could not open the game"; + return false; + } + device.reset(ConstructBlockDevice(loader.get(), error)); + if (!device) { + return false; + } + size = (uint64_t)device->GetNumBlocks() * SECTOR_SIZE; + if (device->GetUncompressedSize() != size) { + *error = "The image isn't a whole number of sectors"; + return false; + } + if (size == 0) { + *error = "The image is empty"; + return false; + } + return true; + } +}; + +bool WriteAll(FILE *f, const void *data, size_t size) { + return size == 0 || fwrite(data, 1, size, f) == size; +} + +// Fails a conversion, removing what it wrote. +bool Fail(FILE *&f, const Path &output, std::string *error, const char *message) { + if (f) { + fclose(f); + f = nullptr; + } + File::Delete(output, true); + if (message) { + *error = message; + } + return false; +} + +} // namespace + +bool Compress(const Path &input, const Path &output, Preset preset, const ProgressFunc &progress, std::string *error, bool *cancelled) { + *cancelled = false; + if (File::Exists(output)) { + *error = "Output file already exists"; + return false; + } + Source src; + if (!src.Open(input, error)) { + return false; + } + + const PresetSettings settings = SettingsFor(preset); + const uint32_t blockSize = settings.blockSize; + const uint32_t sectorsPerBlock = blockSize / SECTOR_SIZE; + const uint32_t numBlocks = (uint32_t)((src.size + blockSize - 1) / blockSize); + const uint32_t numSectors = src.device->GetNumBlocks(); + + // The index holds offsets >> shift in 31 bits. Like maxcso, size it for the worst case, every + // block stored uncompressed. + const uint64_t headerSize = 24; + const uint64_t dataStart = headerSize + ((uint64_t)numBlocks + 1) * 4; + const uint64_t worst = dataStart + src.size; + uint8_t indexShift = 0; + for (int i = 62; i >= 31; --i) { + if (worst >= (1ULL << i)) { + indexShift = (uint8_t)(i + 1 - 31); + break; + } + } + const uint64_t align = 1ULL << indexShift; + auto alignUp = [align](uint64_t pos) { + return (pos + align - 1) & ~(align - 1); + }; + + FILE *f = File::OpenCFile(output, "wb"); + if (!f) { + *error = "Could not create the output file"; + return false; + } + std::vector index(numBlocks + 1); + // Header and index go in at the end, once the offsets are known. + std::vector zeros((size_t)(alignUp(dataStart)), 0); + if (!WriteAll(f, zeros.data(), zeros.size())) { + return Fail(f, output, error, "Could not write the output file"); + } + uint64_t pos = zeros.size(); + + const unsigned threads = std::max(1u, std::thread::hardware_concurrency()); + const uint32_t batchBlocks = std::max(threads * 4, (uint32_t)(BATCH_BYTES / blockSize)); + std::vector in((size_t)batchBlocks * blockSize); + std::vector> out(batchBlocks); + static const u8 padding[2048]{}; + + INFO_LOG(Log::Loader, "CSOCompress: %s -> %s, %u blocks of %u, %u threads", input.c_str(), output.c_str(), numBlocks, blockSize, threads); + for (uint32_t first = 0; first < numBlocks; first += batchBlocks) { + if (progress && !progress((float)first / (float)numBlocks)) { + *cancelled = true; + return Fail(f, output, error, "Cancelled"); + } + const uint32_t count = std::min(batchBlocks, numBlocks - first); + + // The last block past the end of the image is zero-padded, like maxcso does. + const uint32_t firstSector = first * sectorsPerBlock; + const uint32_t sectors = std::min(count * sectorsPerBlock, numSectors - firstSector); + memset(in.data() + (size_t)sectors * SECTOR_SIZE, 0, ((size_t)count * sectorsPerBlock - sectors) * SECTOR_SIZE); + if (!src.device->ReadBlocks(firstSector, (int)sectors, in.data())) { + return Fail(f, output, error, "Could not read the game"); + } + + std::atomic next{0}; + auto work = [&]() { + BlockCompressor compressor(settings); + uint32_t i; + while ((i = next++) < count) { + compressor.Compress(in.data() + (size_t)i * blockSize, &out[i]); + } + }; + std::vector pool; + const unsigned workers = std::min(threads, count); + for (unsigned t = 1; t < workers; ++t) { + pool.emplace_back([&]() { + SetCurrentThreadName("CSOCompress"); + work(); + }); + } + work(); + for (std::thread &t : pool) { + t.join(); + } + + for (uint32_t i = 0; i < count; ++i) { + // Not smaller once padded to the index alignment (images over 2 GB): store it. + const bool stored = out[i].empty() || alignUp(out[i].size()) >= blockSize; + const u8 *data = stored ? in.data() + (size_t)i * blockSize : out[i].data(); + const size_t size = stored ? blockSize : out[i].size(); + index[first + i] = (u32)(pos >> indexShift) | (stored ? CSO_INDEX_UNCOMPRESSED : 0); + if (!WriteAll(f, data, size)) { + return Fail(f, output, error, "Could not write the output file"); + } + pos += size; + const uint64_t aligned = alignUp(pos); + if (!WriteAll(f, padding, (size_t)(aligned - pos))) { + return Fail(f, output, error, "Could not write the output file"); + } + pos = aligned; + } + } + index[numBlocks] = (u32)(pos >> indexShift); + + // The header: "CISO", its size, the image size, the block size, version 1 and the shift. + u8 header[24]{}; + memcpy(header, "CISO", 4); + const u32_le headerSizeLE = (u32)headerSize; + const u64_le sizeLE = src.size; + const u32_le blockSizeLE = blockSize; + memcpy(header + 4, &headerSizeLE, 4); + memcpy(header + 8, &sizeLE, 8); + memcpy(header + 16, &blockSizeLE, 4); + header[20] = 1; + header[21] = indexShift; + if (File::Fseek(f, 0, SEEK_SET) != 0 || !WriteAll(f, header, sizeof(header)) || !WriteAll(f, index.data(), index.size() * sizeof(u32_le))) { + return Fail(f, output, error, "Could not write the output file"); + } + if (fclose(f) != 0) { + f = nullptr; + return Fail(f, output, error, "Could not write the output file"); + } + if (progress) { + progress(1.0f); + } + return true; +} + +bool Extract(const Path &input, const Path &output, const ProgressFunc &progress, std::string *error, bool *cancelled) { + *cancelled = false; + if (File::Exists(output)) { + *error = "Output file already exists"; + return false; + } + Source src; + if (!src.Open(input, error)) { + return false; + } + FILE *f = File::OpenCFile(output, "wb"); + if (!f) { + *error = "Could not create the output file"; + return false; + } + + const uint32_t numSectors = src.device->GetNumBlocks(); + const uint32_t batch = (uint32_t)(BATCH_BYTES / SECTOR_SIZE); + std::vector buffer((size_t)batch * SECTOR_SIZE); + for (uint32_t first = 0; first < numSectors; first += batch) { + if (progress && !progress((float)first / (float)numSectors)) { + *cancelled = true; + return Fail(f, output, error, "Cancelled"); + } + const uint32_t count = std::min(batch, numSectors - first); + if (!src.device->ReadBlocks(first, (int)count, buffer.data())) { + return Fail(f, output, error, "Could not read the game"); + } + if (!WriteAll(f, buffer.data(), (size_t)count * SECTOR_SIZE)) { + return Fail(f, output, error, "Could not write the output file"); + } + } + if (fclose(f) != 0) { + f = nullptr; + return Fail(f, output, error, "Could not write the output file"); + } + if (progress) { + progress(1.0f); + } + return true; +} + +} // namespace CSOCompress diff --git a/Core/FileSystems/CSOCompress.h b/Core/FileSystems/CSOCompress.h new file mode 100644 index 0000000000..16247124d4 --- /dev/null +++ b/Core/FileSystems/CSOCompress.h @@ -0,0 +1,43 @@ +// Copyright (c) 2012- PPSSPP Project. + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, version 2.0 or later versions. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License 2.0 for more details. + +// A copy of the GPL 2.0 should have been included with the program. +// If not, see http://www.gnu.org/licenses/ + +// Official git repository and contact information can be found at +// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. + +#pragma once + +#include +#include + +#include "Common/File/Path.h" + +// Writes PSP disc images as CSO v1 or plain ISO. The input is anything PPSSPP can play as a disc +// (ISO, CSO, CHD, a PSN EBOOT.PBP), read through its own block devices, so a repack is one pass +// with no temporary file. The presets match maxcso's (LunaMoo/maxcso). +namespace CSOCompress { + +enum class Preset { + Best = 0, // libdeflate level 12, 64K blocks. Small, fast to compress and quick to read. + PSPCompatible = 1, // zlib level 9, 2K blocks: what every PSP custom firmware reads. +}; + +// Return false to cancel. +typedef std::function ProgressFunc; + +// Blocking, and uses every core. Never overwrites, and deletes a partial output on failure or +// cancel (*cancelled is set then). +bool Compress(const Path &input, const Path &output, Preset preset, const ProgressFunc &progress, std::string *error, bool *cancelled); +bool Extract(const Path &input, const Path &output, const ProgressFunc &progress, std::string *error, bool *cancelled); + +} // namespace CSOCompress diff --git a/UI/CMakeLists.txt b/UI/CMakeLists.txt index a9f0b09ad5..9354bf4e16 100644 --- a/UI/CMakeLists.txt +++ b/UI/CMakeLists.txt @@ -63,6 +63,8 @@ list(APPEND UISource TabbedDialogScreen.cpp GameScreen.h GameScreen.cpp + GamePacker.h + GamePacker.cpp GameSettingsScreen.h GameSettingsScreen.cpp DeveloperToolsScreen.h diff --git a/UI/GameBrowser.cpp b/UI/GameBrowser.cpp index ac58ebdb5b..818f2ef5fc 100644 --- a/UI/GameBrowser.cpp +++ b/UI/GameBrowser.cpp @@ -41,6 +41,7 @@ #include "Core/Util/VideoPlayer.h" #include "UI/GameBrowser.h" +#include "UI/GamePacker.h" #include "UI/RemoteISOScreen.h" #include "UI/SavedataScreen.h" #include "UI/Store.h" @@ -659,6 +660,20 @@ void GameButton::Draw(UIContext &dc) { dc.FillRect(Drawable(overlayColor), overlayBounds); } + // Covers the icon while the game is being compressed. Clicking opens the game screen instead of playing. + GamePacker::Status packStatus; + if (GamePacker::IsPacking(gamePath_) && GamePacker::GetStatus(&packStatus)) { + auto ga = GetI18NCategory(I18NCat::GAME); + const Bounds iconBounds(x, y, w, h); + dc.FillRect(Drawable(0xD0505050), iconBounds); + const float scale = gridStyle_ || largeList ? 0.8f * g_Config.fGameGridScale : 0.6f; + dc.SetFontStyle(dc.GetTheme().uiFont); + dc.SetFontScale(scale, scale); + const std::string text = std::string(ga->T("Compressing...")) + StringFromFormat("\n%d%%", (int)(packStatus.progress * 100.0f)); + dc.DrawTextRectSqueeze(text, iconBounds, 0xFFFFFFFF, ALIGN_CENTER); + dc.SetFontScale(1.0f, 1.0f); + } + dc.RebindTexture(); } diff --git a/UI/GamePacker.cpp b/UI/GamePacker.cpp new file mode 100644 index 0000000000..83c146ce5f --- /dev/null +++ b/UI/GamePacker.cpp @@ -0,0 +1,478 @@ +// Copyright (c) 2012- PPSSPP Project. + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, version 2.0 or later versions. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License 2.0 for more details. + +// A copy of the GPL 2.0 should have been included with the program. +// If not, see http://www.gnu.org/licenses/ + +// Official git repository and contact information can be found at +// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. + +#include +#include +#include +#include + +#include "Common/Data/Color/RGBAUtil.h" +#include "Common/Data/Text/I18n.h" +#include "Common/Data/Text/Parsers.h" +#include "Common/File/FileUtil.h" +#include "Common/Log.h" +#include "Common/StringUtils.h" +#include "Common/System/OSD.h" +#include "Common/Thread/ThreadUtil.h" +#include "Common/TimeUtil.h" +#include "Common/UI/Context.h" + +#include "Core/Config.h" +#include "Core/System.h" +#include "Core/Util/RecentFiles.h" + +#include "UI/BackgroundAudio.h" +#include "UI/GameInfoCache.h" +#include "UI/GamePacker.h" + +namespace GamePacker { +namespace { + +struct Job { + Path gamePath; // As the game list knows it, maybe a PSP/GAME/. + Path file; // The image itself. + Path finalOutput; // Where the converted game ends up. + Path writeOutput; // What gets written: finalOutput, or a temporary name when that's the original's. + bool replace = false; // Delete the original when done. + bool toISO = false; + CSOCompress::Preset preset = CSOCompress::Preset::Best; + + std::atomic progress{0.0f}; + double startTime = 0.0; + std::atomic cancel{false}; + std::atomic done{false}; + bool success = false; // Set before done. + bool cancelled = false; // Set before done. + std::string error; // Set before done. + + std::thread thread; +}; + +// Only touched on the UI thread. The worker gets the raw pointer, and the job lives until it's joined. +std::unique_ptr g_job; +int g_generation = 0; +Result g_lastResult; + +void RunJob(Job *job) { + SetCurrentThreadName("GamePacker"); + INFO_LOG(Log::Loader, "GamePacker: %s %s -> %s", job->toISO ? "Extracting" : "Compressing", job->file.c_str(), job->writeOutput.c_str()); + + auto progress = [job](float p) { + job->progress = p; + return !job->cancel; + }; + std::string error; + bool cancelled = false; + bool success; + if (job->toISO) { + success = CSOCompress::Extract(job->file, job->writeOutput, progress, &error, &cancelled); + } else { + success = CSOCompress::Compress(job->file, job->writeOutput, job->preset, progress, &error, &cancelled); + } + if (!success && !cancelled) { + ERROR_LOG(Log::Loader, "GamePacker: Failed on %s: %s", job->file.c_str(), error.c_str()); + } + job->success = success; + job->cancelled = cancelled; + job->error = error; + job->done = true; +} + +// "" for ".", or for an EBOOT.PBP the game's disc id. +std::string OutputBaseName(const Path &gamePath, const Path &file) { + std::string name = file.GetFilename(); + const std::string ext = file.GetFileExtension(); + if (ext.size() <= name.size()) { + name.resize(name.size() - ext.size()); + } + if (equalsNoCase(file.GetFilename(), "EBOOT.PBP")) { + std::shared_ptr info = g_gameInfoCache->GetInfo(nullptr, gamePath, GameInfoFlags::PARAM_SFO); + if (info->Ready(GameInfoFlags::PARAM_SFO)) { + const std::string id = SanitizeString(info->id, StringRestriction::FileName); + if (!id.empty()) { + return id; + } + } + } + return name; +} + +// Where an option puts its result, before looking at what's already there. +Path FinalOutput(const Path &gamePath, const Path &file, Container from, const Option &option) { + const Container to = option.toISO ? Container::ISO : Container::CSO; + if (to == from) { + return file; + } + return file.NavigateUp() / (OutputBaseName(gamePath, file) + (option.toISO ? ".iso" : ".cso")); +} + +// " (2).", " (3)."... for a name that's taken. +Path NumberedName(const Path &path) { + const std::string ext = path.GetFileExtension(); + std::string name = path.GetFilename(); + name.resize(name.size() - ext.size()); + for (int i = 2; i < 1000; i++) { + Path candidate = path.NavigateUp() / StringFromFormat("%s (%d)%s", name.c_str(), i, ext.c_str()); + if (!File::Exists(candidate)) { + return candidate; + } + } + return Path(); +} + +} // namespace + +Container DetectContainer(const Path &gamePath, Path *file) { + // Content URIs are fine too (Android's storage), only remote games aren't. + if (gamePath.Type() == PathType::HTTP) { + return Container::None; + } + Path image = gamePath; + if (File::IsDirectory(gamePath)) { + image = gamePath / "EBOOT.PBP"; + if (!File::Exists(image)) { + return Container::None; + } + } + const std::string ext = image.GetFileExtension(); + Container container = Container::None; + if (ext == ".iso") { + container = Container::ISO; + } else if (ext == ".cso") { + container = Container::CSO; + } else if (ext == ".chd") { + container = Container::CHD; + } else if (ext == ".pbp") { + container = Container::PBP; + } + if (container != Container::None && file) { + *file = image; + } + return container; +} + +std::vector