diff --git a/Core/CMakeLists.txt b/Core/CMakeLists.txt
index 4514429bfc..1c9a19417f 100644
--- a/Core/CMakeLists.txt
+++ b/Core/CMakeLists.txt
@@ -357,6 +357,8 @@ add_library(Core STATIC
FileSystems/BlobFileSystem.h
FileSystems/BlockDevices.cpp
FileSystems/BlockDevices.h
+ FileSystems/CSOCompress.cpp
+ FileSystems/CSOCompress.h
FileSystems/DirectoryFileSystem.cpp
FileSystems/DirectoryFileSystem.h
FileSystems/FileSystem.h
@@ -713,7 +715,7 @@ endif()
list(APPEND CoreExtraLibs armips)
-target_link_libraries(Core GPU Common chdr kirk cityhash sfmt19937 xbrz xxhash rcheevos at3_standalone lua
+target_link_libraries(Core GPU Common chdr kirk cityhash sfmt19937 xbrz xxhash libdeflate rcheevos at3_standalone lua
aemu_postoffice_client ${CoreExtraLibs} ${CMAKE_DL_LIBS})
# Winsock
diff --git a/Core/Config.cpp b/Core/Config.cpp
index 7edd3af46f..15318c5edc 100644
--- a/Core/Config.cpp
+++ b/Core/Config.cpp
@@ -331,6 +331,8 @@ static const ConfigSetting generalSettings[] = {
ConfigSetting("CompressSymbols", SETTING(g_Config, bCompressSymbols), true, CfgFlag::DEFAULT),
ConfigSetting("CacheFullIsoInRam", SETTING(g_Config, bCacheFullIsoInRam), false, CfgFlag::PER_GAME),
ConfigSetting("AutoUpgradeFirmware", SETTING(g_Config, bAutoUpgradeFirmware), true, CfgFlag::DEFAULT),
+ ConfigSetting("CSOPreset", SETTING(g_Config, iCSOPreset), 0, CfgFlag::DEFAULT),
+ ConfigSetting("PackKeepOriginal", SETTING(g_Config, bPackKeepOriginal), false, CfgFlag::DEFAULT),
ConfigSetting("RemoteISOPort", SETTING(g_Config, iRemoteISOPort), 0, CfgFlag::DEFAULT),
ConfigSetting("LastRemoteISOServer", SETTING(g_Config, sLastRemoteISOServer), "", CfgFlag::DEFAULT),
ConfigSetting("LastRemoteISOPort", SETTING(g_Config, iLastRemoteISOPort), 0, CfgFlag::DEFAULT),
diff --git a/Core/Config.h b/Core/Config.h
index 7705d27b43..2dac232622 100644
--- a/Core/Config.h
+++ b/Core/Config.h
@@ -251,6 +251,9 @@ public:
bool bCompressSymbols;
bool bCacheFullIsoInRam;
bool bAutoUpgradeFirmware;
+ // The game screen's "Compress game" (UI/GamePacker.h).
+ int iCSOPreset; // CSOCompress::Preset: 0 = Best, 1 = PSP Compatible
+ bool bPackKeepOriginal;
int iRemoteISOPort; // Also used for serving a local remote debugger.
std::string sLastRemoteISOServer;
int iLastRemoteISOPort;
diff --git a/Core/Core.vcxproj b/Core/Core.vcxproj
index aaadb94767..c413dd7ed7 100644
--- a/Core/Core.vcxproj
+++ b/Core/Core.vcxproj
@@ -585,6 +585,7 @@
+
@@ -738,6 +739,8 @@
+
+ MaxSpeedtrue
@@ -919,6 +922,7 @@
+
@@ -1060,6 +1064,19 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/Core/Core.vcxproj.filters b/Core/Core.vcxproj.filters
index 5c5604c4e8..1ec7c1115c 100644
--- a/Core/Core.vcxproj.filters
+++ b/Core/Core.vcxproj.filters
@@ -82,6 +82,9 @@
{6c6e0188-fabf-4203-aaa2-896c42be2c20}
+
+ {5b0c7e1a-3d2f-4c8e-9a61-d4f0b2e7c913}
+ {0201ff27-4fa8-4cf3-bbb9-accfdc0c7046}
@@ -141,6 +144,9 @@
FileSystems
+
+ FileSystems
+ FileSystems
@@ -502,6 +508,12 @@
MIPS
+
+ Ext\libdeflate
+
+
+ Ext\libdeflate
+ Ext
@@ -1422,6 +1434,9 @@
FileSystems
+
+ FileSystems
+ FileSystems
@@ -1779,6 +1794,45 @@
Ext
+
+ Ext\libdeflate
+
+
+ Ext\libdeflate
+
+
+ Ext\libdeflate
+
+
+ Ext\libdeflate
+
+
+ Ext\libdeflate
+
+
+ Ext\libdeflate
+
+
+ Ext\libdeflate
+
+
+ Ext\libdeflate
+
+
+ Ext\libdeflate
+
+
+ Ext\libdeflate
+
+
+ Ext\libdeflate
+
+
+ Ext\libdeflate
+
+
+ Ext\libdeflate
+ Ext
diff --git a/Core/FileSystems/CSOCompress.cpp b/Core/FileSystems/CSOCompress.cpp
new file mode 100644
index 0000000000..1337efd5b2
--- /dev/null
+++ b/Core/FileSystems/CSOCompress.cpp
@@ -0,0 +1,354 @@
+// Copyright (c) 2012- PPSSPP Project.
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, version 2.0 or later versions.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License 2.0 for more details.
+
+// A copy of the GPL 2.0 should have been included with the program.
+// If not, see http://www.gnu.org/licenses/
+
+// Official git repository and contact information can be found at
+// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/.
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "Common/CommonTypes.h"
+#include "Common/File/FileUtil.h"
+#include "Common/Log.h"
+#include "Common/Swap.h"
+#include "Common/Thread/ThreadUtil.h"
+#include "Core/FileSystems/BlockDevices.h"
+#include "Core/Loaders.h"
+#include "Core/FileSystems/CSOCompress.h"
+
+#ifdef SHARED_ZLIB
+#include
+#else
+#include "ext/zlib/zlib.h"
+#endif
+
+#include "ext/libdeflate/libdeflate.h"
+
+namespace CSOCompress {
+
+namespace {
+
+const uint32_t SECTOR_SIZE = 2048;
+const uint32_t CSO_INDEX_UNCOMPRESSED = 0x80000000;
+// Input read and compressed per round, so the threads always have work queued.
+const size_t BATCH_BYTES = 32 * 1024 * 1024;
+
+struct PresetSettings {
+ uint32_t blockSize;
+ bool zlib; // zlib level 9, default strategy.
+ bool libdeflate; // libdeflate level 12.
+};
+
+// The trials maxcso's presets ran (with lz4, 7-zip and zlib's brute force strategies left out).
+PresetSettings SettingsFor(Preset preset) {
+ switch (preset) {
+ case Preset::Best:
+ // 64K: only ~0.25% bigger than 256K, and each read of a new block inflates a quarter as much.
+ return { 65536, false, true };
+ case Preset::PSPCompatible:
+ default:
+ // Some custom firmware can't read libdeflate's output or larger blocks.
+ return { 2048, true, false };
+ }
+}
+
+// One thread's encoders, reused for every block it compresses.
+class BlockCompressor {
+public:
+ explicit BlockCompressor(const PresetSettings &settings) : settings_(settings), scratch_(settings.blockSize) {
+ if (settings_.zlib) {
+ zlibOk_ = deflateInit2(&z_, 9, Z_DEFLATED, -15, 9, Z_DEFAULT_STRATEGY) == Z_OK;
+ }
+ if (settings_.libdeflate) {
+ libdeflate_ = libdeflate_alloc_compressor(12);
+ }
+ }
+ ~BlockCompressor() {
+ if (zlibOk_) {
+ deflateEnd(&z_);
+ }
+ if (libdeflate_) {
+ libdeflate_free_compressor(libdeflate_);
+ }
+ }
+
+ // Leaves the smallest raw deflate stream in *out, or nothing when none beats the block itself.
+ void Compress(const u8 *in, std::vector *out) {
+ const uint32_t blockSize = settings_.blockSize;
+ out->clear();
+ auto submit = [&](const u8 *data, size_t size) {
+ // Not smaller than the block: stored as is, which also reads faster.
+ if (size > 0 && size < blockSize && (out->empty() || size < out->size())) {
+ out->assign(data, data + size);
+ }
+ };
+
+ if (libdeflate_) {
+ // libdeflate returns 0 when the result doesn't fit, i.e. isn't smaller.
+ const size_t size = libdeflate_deflate_compress(libdeflate_, in, blockSize, scratch_.data(), scratch_.size());
+ submit(scratch_.data(), size);
+ }
+ if (zlibOk_ && deflateReset(&z_) == Z_OK) {
+ z_.next_in = (Bytef *)in;
+ z_.avail_in = blockSize;
+ z_.next_out = scratch_.data();
+ z_.avail_out = (uInt)scratch_.size();
+ int res;
+ while ((res = deflate(&z_, Z_FINISH)) == Z_OK) {
+ continue;
+ }
+ if (res == Z_STREAM_END) {
+ submit(scratch_.data(), z_.total_out);
+ }
+ }
+ }
+
+private:
+ PresetSettings settings_;
+ std::vector scratch_;
+ z_stream z_{};
+ bool zlibOk_ = false;
+ libdeflate_compressor *libdeflate_ = nullptr;
+};
+
+// Opens anything PPSSPP plays as a disc, as 2048-byte sectors.
+struct Source {
+ std::unique_ptr loader;
+ std::unique_ptr device;
+ uint64_t size = 0;
+
+ bool Open(const Path &path, std::string *error) {
+ loader.reset(ConstructFileLoader(path));
+ if (!loader || !loader->Exists()) {
+ *error = "Could not open the game";
+ return false;
+ }
+ device.reset(ConstructBlockDevice(loader.get(), error));
+ if (!device) {
+ return false;
+ }
+ size = (uint64_t)device->GetNumBlocks() * SECTOR_SIZE;
+ if (device->GetUncompressedSize() != size) {
+ *error = "The image isn't a whole number of sectors";
+ return false;
+ }
+ if (size == 0) {
+ *error = "The image is empty";
+ return false;
+ }
+ return true;
+ }
+};
+
+bool WriteAll(FILE *f, const void *data, size_t size) {
+ return size == 0 || fwrite(data, 1, size, f) == size;
+}
+
+// Fails a conversion, removing what it wrote.
+bool Fail(FILE *&f, const Path &output, std::string *error, const char *message) {
+ if (f) {
+ fclose(f);
+ f = nullptr;
+ }
+ File::Delete(output, true);
+ if (message) {
+ *error = message;
+ }
+ return false;
+}
+
+} // namespace
+
+bool Compress(const Path &input, const Path &output, Preset preset, const ProgressFunc &progress, std::string *error, bool *cancelled) {
+ *cancelled = false;
+ if (File::Exists(output)) {
+ *error = "Output file already exists";
+ return false;
+ }
+ Source src;
+ if (!src.Open(input, error)) {
+ return false;
+ }
+
+ const PresetSettings settings = SettingsFor(preset);
+ const uint32_t blockSize = settings.blockSize;
+ const uint32_t sectorsPerBlock = blockSize / SECTOR_SIZE;
+ const uint32_t numBlocks = (uint32_t)((src.size + blockSize - 1) / blockSize);
+ const uint32_t numSectors = src.device->GetNumBlocks();
+
+ // The index holds offsets >> shift in 31 bits. Like maxcso, size it for the worst case, every
+ // block stored uncompressed.
+ const uint64_t headerSize = 24;
+ const uint64_t dataStart = headerSize + ((uint64_t)numBlocks + 1) * 4;
+ const uint64_t worst = dataStart + src.size;
+ uint8_t indexShift = 0;
+ for (int i = 62; i >= 31; --i) {
+ if (worst >= (1ULL << i)) {
+ indexShift = (uint8_t)(i + 1 - 31);
+ break;
+ }
+ }
+ const uint64_t align = 1ULL << indexShift;
+ auto alignUp = [align](uint64_t pos) {
+ return (pos + align - 1) & ~(align - 1);
+ };
+
+ FILE *f = File::OpenCFile(output, "wb");
+ if (!f) {
+ *error = "Could not create the output file";
+ return false;
+ }
+ std::vector index(numBlocks + 1);
+ // Header and index go in at the end, once the offsets are known.
+ std::vector zeros((size_t)(alignUp(dataStart)), 0);
+ if (!WriteAll(f, zeros.data(), zeros.size())) {
+ return Fail(f, output, error, "Could not write the output file");
+ }
+ uint64_t pos = zeros.size();
+
+ const unsigned threads = std::max(1u, std::thread::hardware_concurrency());
+ const uint32_t batchBlocks = std::max(threads * 4, (uint32_t)(BATCH_BYTES / blockSize));
+ std::vector in((size_t)batchBlocks * blockSize);
+ std::vector> out(batchBlocks);
+ static const u8 padding[2048]{};
+
+ INFO_LOG(Log::Loader, "CSOCompress: %s -> %s, %u blocks of %u, %u threads", input.c_str(), output.c_str(), numBlocks, blockSize, threads);
+ for (uint32_t first = 0; first < numBlocks; first += batchBlocks) {
+ if (progress && !progress((float)first / (float)numBlocks)) {
+ *cancelled = true;
+ return Fail(f, output, error, "Cancelled");
+ }
+ const uint32_t count = std::min(batchBlocks, numBlocks - first);
+
+ // The last block past the end of the image is zero-padded, like maxcso does.
+ const uint32_t firstSector = first * sectorsPerBlock;
+ const uint32_t sectors = std::min(count * sectorsPerBlock, numSectors - firstSector);
+ memset(in.data() + (size_t)sectors * SECTOR_SIZE, 0, ((size_t)count * sectorsPerBlock - sectors) * SECTOR_SIZE);
+ if (!src.device->ReadBlocks(firstSector, (int)sectors, in.data())) {
+ return Fail(f, output, error, "Could not read the game");
+ }
+
+ std::atomic next{0};
+ auto work = [&]() {
+ BlockCompressor compressor(settings);
+ uint32_t i;
+ while ((i = next++) < count) {
+ compressor.Compress(in.data() + (size_t)i * blockSize, &out[i]);
+ }
+ };
+ std::vector pool;
+ const unsigned workers = std::min(threads, count);
+ for (unsigned t = 1; t < workers; ++t) {
+ pool.emplace_back([&]() {
+ SetCurrentThreadName("CSOCompress");
+ work();
+ });
+ }
+ work();
+ for (std::thread &t : pool) {
+ t.join();
+ }
+
+ for (uint32_t i = 0; i < count; ++i) {
+ // Not smaller once padded to the index alignment (images over 2 GB): store it.
+ const bool stored = out[i].empty() || alignUp(out[i].size()) >= blockSize;
+ const u8 *data = stored ? in.data() + (size_t)i * blockSize : out[i].data();
+ const size_t size = stored ? blockSize : out[i].size();
+ index[first + i] = (u32)(pos >> indexShift) | (stored ? CSO_INDEX_UNCOMPRESSED : 0);
+ if (!WriteAll(f, data, size)) {
+ return Fail(f, output, error, "Could not write the output file");
+ }
+ pos += size;
+ const uint64_t aligned = alignUp(pos);
+ if (!WriteAll(f, padding, (size_t)(aligned - pos))) {
+ return Fail(f, output, error, "Could not write the output file");
+ }
+ pos = aligned;
+ }
+ }
+ index[numBlocks] = (u32)(pos >> indexShift);
+
+ // The header: "CISO", its size, the image size, the block size, version 1 and the shift.
+ u8 header[24]{};
+ memcpy(header, "CISO", 4);
+ const u32_le headerSizeLE = (u32)headerSize;
+ const u64_le sizeLE = src.size;
+ const u32_le blockSizeLE = blockSize;
+ memcpy(header + 4, &headerSizeLE, 4);
+ memcpy(header + 8, &sizeLE, 8);
+ memcpy(header + 16, &blockSizeLE, 4);
+ header[20] = 1;
+ header[21] = indexShift;
+ if (File::Fseek(f, 0, SEEK_SET) != 0 || !WriteAll(f, header, sizeof(header)) || !WriteAll(f, index.data(), index.size() * sizeof(u32_le))) {
+ return Fail(f, output, error, "Could not write the output file");
+ }
+ if (fclose(f) != 0) {
+ f = nullptr;
+ return Fail(f, output, error, "Could not write the output file");
+ }
+ if (progress) {
+ progress(1.0f);
+ }
+ return true;
+}
+
+bool Extract(const Path &input, const Path &output, const ProgressFunc &progress, std::string *error, bool *cancelled) {
+ *cancelled = false;
+ if (File::Exists(output)) {
+ *error = "Output file already exists";
+ return false;
+ }
+ Source src;
+ if (!src.Open(input, error)) {
+ return false;
+ }
+ FILE *f = File::OpenCFile(output, "wb");
+ if (!f) {
+ *error = "Could not create the output file";
+ return false;
+ }
+
+ const uint32_t numSectors = src.device->GetNumBlocks();
+ const uint32_t batch = (uint32_t)(BATCH_BYTES / SECTOR_SIZE);
+ std::vector buffer((size_t)batch * SECTOR_SIZE);
+ for (uint32_t first = 0; first < numSectors; first += batch) {
+ if (progress && !progress((float)first / (float)numSectors)) {
+ *cancelled = true;
+ return Fail(f, output, error, "Cancelled");
+ }
+ const uint32_t count = std::min(batch, numSectors - first);
+ if (!src.device->ReadBlocks(first, (int)count, buffer.data())) {
+ return Fail(f, output, error, "Could not read the game");
+ }
+ if (!WriteAll(f, buffer.data(), (size_t)count * SECTOR_SIZE)) {
+ return Fail(f, output, error, "Could not write the output file");
+ }
+ }
+ if (fclose(f) != 0) {
+ f = nullptr;
+ return Fail(f, output, error, "Could not write the output file");
+ }
+ if (progress) {
+ progress(1.0f);
+ }
+ return true;
+}
+
+} // namespace CSOCompress
diff --git a/Core/FileSystems/CSOCompress.h b/Core/FileSystems/CSOCompress.h
new file mode 100644
index 0000000000..16247124d4
--- /dev/null
+++ b/Core/FileSystems/CSOCompress.h
@@ -0,0 +1,43 @@
+// Copyright (c) 2012- PPSSPP Project.
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, version 2.0 or later versions.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License 2.0 for more details.
+
+// A copy of the GPL 2.0 should have been included with the program.
+// If not, see http://www.gnu.org/licenses/
+
+// Official git repository and contact information can be found at
+// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/.
+
+#pragma once
+
+#include
+#include
+
+#include "Common/File/Path.h"
+
+// Writes PSP disc images as CSO v1 or plain ISO. The input is anything PPSSPP can play as a disc
+// (ISO, CSO, CHD, a PSN EBOOT.PBP), read through its own block devices, so a repack is one pass
+// with no temporary file. The presets match maxcso's (LunaMoo/maxcso).
+namespace CSOCompress {
+
+enum class Preset {
+ Best = 0, // libdeflate level 12, 64K blocks. Small, fast to compress and quick to read.
+ PSPCompatible = 1, // zlib level 9, 2K blocks: what every PSP custom firmware reads.
+};
+
+// Return false to cancel.
+typedef std::function ProgressFunc;
+
+// Blocking, and uses every core. Never overwrites, and deletes a partial output on failure or
+// cancel (*cancelled is set then).
+bool Compress(const Path &input, const Path &output, Preset preset, const ProgressFunc &progress, std::string *error, bool *cancelled);
+bool Extract(const Path &input, const Path &output, const ProgressFunc &progress, std::string *error, bool *cancelled);
+
+} // namespace CSOCompress
diff --git a/UI/CMakeLists.txt b/UI/CMakeLists.txt
index a9f0b09ad5..9354bf4e16 100644
--- a/UI/CMakeLists.txt
+++ b/UI/CMakeLists.txt
@@ -63,6 +63,8 @@ list(APPEND UISource
TabbedDialogScreen.cpp
GameScreen.h
GameScreen.cpp
+ GamePacker.h
+ GamePacker.cpp
GameSettingsScreen.h
GameSettingsScreen.cpp
DeveloperToolsScreen.h
diff --git a/UI/GameBrowser.cpp b/UI/GameBrowser.cpp
index 579518a9c1..486edcae11 100644
--- a/UI/GameBrowser.cpp
+++ b/UI/GameBrowser.cpp
@@ -41,6 +41,7 @@
#include "Core/Util/VideoPlayer.h"
#include "UI/GameBrowser.h"
+#include "UI/GamePacker.h"
#include "UI/RemoteISOScreen.h"
#include "UI/SavedataScreen.h"
#include "UI/Store.h"
@@ -660,6 +661,20 @@ void GameButton::Draw(UIContext &dc) {
dc.FillRect(Drawable(overlayColor), overlayBounds);
}
+ // Covers the icon while the game is being compressed. Clicking opens the game screen instead of playing.
+ GamePacker::Status packStatus;
+ if (GamePacker::IsPacking(gamePath_) && GamePacker::GetStatus(&packStatus)) {
+ auto ga = GetI18NCategory(I18NCat::GAME);
+ const Bounds iconBounds(x, y, w, h);
+ dc.FillRect(Drawable(0xD0505050), iconBounds);
+ const float scale = gridStyle_ || largeList ? 0.8f * g_Config.fGameGridScale : 0.6f;
+ dc.SetFontStyle(dc.GetTheme().uiFont);
+ dc.SetFontScale(scale, scale);
+ const std::string text = std::string(ga->T("Compressing...")) + StringFromFormat("\n%d%%", (int)(packStatus.progress * 100.0f));
+ dc.DrawTextRectSqueeze(text, iconBounds, 0xFFFFFFFF, ALIGN_CENTER);
+ dc.SetFontScale(1.0f, 1.0f);
+ }
+
dc.RebindTexture();
}
diff --git a/UI/GamePacker.cpp b/UI/GamePacker.cpp
new file mode 100644
index 0000000000..83c146ce5f
--- /dev/null
+++ b/UI/GamePacker.cpp
@@ -0,0 +1,478 @@
+// Copyright (c) 2012- PPSSPP Project.
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, version 2.0 or later versions.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License 2.0 for more details.
+
+// A copy of the GPL 2.0 should have been included with the program.
+// If not, see http://www.gnu.org/licenses/
+
+// Official git repository and contact information can be found at
+// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/.
+
+#include
+#include
+#include
+#include
+
+#include "Common/Data/Color/RGBAUtil.h"
+#include "Common/Data/Text/I18n.h"
+#include "Common/Data/Text/Parsers.h"
+#include "Common/File/FileUtil.h"
+#include "Common/Log.h"
+#include "Common/StringUtils.h"
+#include "Common/System/OSD.h"
+#include "Common/Thread/ThreadUtil.h"
+#include "Common/TimeUtil.h"
+#include "Common/UI/Context.h"
+
+#include "Core/Config.h"
+#include "Core/System.h"
+#include "Core/Util/RecentFiles.h"
+
+#include "UI/BackgroundAudio.h"
+#include "UI/GameInfoCache.h"
+#include "UI/GamePacker.h"
+
+namespace GamePacker {
+namespace {
+
+struct Job {
+ Path gamePath; // As the game list knows it, maybe a PSP/GAME/.
+ Path file; // The image itself.
+ Path finalOutput; // Where the converted game ends up.
+ Path writeOutput; // What gets written: finalOutput, or a temporary name when that's the original's.
+ bool replace = false; // Delete the original when done.
+ bool toISO = false;
+ CSOCompress::Preset preset = CSOCompress::Preset::Best;
+
+ std::atomic progress{0.0f};
+ double startTime = 0.0;
+ std::atomic cancel{false};
+ std::atomic done{false};
+ bool success = false; // Set before done.
+ bool cancelled = false; // Set before done.
+ std::string error; // Set before done.
+
+ std::thread thread;
+};
+
+// Only touched on the UI thread. The worker gets the raw pointer, and the job lives until it's joined.
+std::unique_ptr g_job;
+int g_generation = 0;
+Result g_lastResult;
+
+void RunJob(Job *job) {
+ SetCurrentThreadName("GamePacker");
+ INFO_LOG(Log::Loader, "GamePacker: %s %s -> %s", job->toISO ? "Extracting" : "Compressing", job->file.c_str(), job->writeOutput.c_str());
+
+ auto progress = [job](float p) {
+ job->progress = p;
+ return !job->cancel;
+ };
+ std::string error;
+ bool cancelled = false;
+ bool success;
+ if (job->toISO) {
+ success = CSOCompress::Extract(job->file, job->writeOutput, progress, &error, &cancelled);
+ } else {
+ success = CSOCompress::Compress(job->file, job->writeOutput, job->preset, progress, &error, &cancelled);
+ }
+ if (!success && !cancelled) {
+ ERROR_LOG(Log::Loader, "GamePacker: Failed on %s: %s", job->file.c_str(), error.c_str());
+ }
+ job->success = success;
+ job->cancelled = cancelled;
+ job->error = error;
+ job->done = true;
+}
+
+// "" for ".", or for an EBOOT.PBP the game's disc id.
+std::string OutputBaseName(const Path &gamePath, const Path &file) {
+ std::string name = file.GetFilename();
+ const std::string ext = file.GetFileExtension();
+ if (ext.size() <= name.size()) {
+ name.resize(name.size() - ext.size());
+ }
+ if (equalsNoCase(file.GetFilename(), "EBOOT.PBP")) {
+ std::shared_ptr info = g_gameInfoCache->GetInfo(nullptr, gamePath, GameInfoFlags::PARAM_SFO);
+ if (info->Ready(GameInfoFlags::PARAM_SFO)) {
+ const std::string id = SanitizeString(info->id, StringRestriction::FileName);
+ if (!id.empty()) {
+ return id;
+ }
+ }
+ }
+ return name;
+}
+
+// Where an option puts its result, before looking at what's already there.
+Path FinalOutput(const Path &gamePath, const Path &file, Container from, const Option &option) {
+ const Container to = option.toISO ? Container::ISO : Container::CSO;
+ if (to == from) {
+ return file;
+ }
+ return file.NavigateUp() / (OutputBaseName(gamePath, file) + (option.toISO ? ".iso" : ".cso"));
+}
+
+// " (2).", " (3)."... for a name that's taken.
+Path NumberedName(const Path &path) {
+ const std::string ext = path.GetFileExtension();
+ std::string name = path.GetFilename();
+ name.resize(name.size() - ext.size());
+ for (int i = 2; i < 1000; i++) {
+ Path candidate = path.NavigateUp() / StringFromFormat("%s (%d)%s", name.c_str(), i, ext.c_str());
+ if (!File::Exists(candidate)) {
+ return candidate;
+ }
+ }
+ return Path();
+}
+
+} // namespace
+
+Container DetectContainer(const Path &gamePath, Path *file) {
+ // Content URIs are fine too (Android's storage), only remote games aren't.
+ if (gamePath.Type() == PathType::HTTP) {
+ return Container::None;
+ }
+ Path image = gamePath;
+ if (File::IsDirectory(gamePath)) {
+ image = gamePath / "EBOOT.PBP";
+ if (!File::Exists(image)) {
+ return Container::None;
+ }
+ }
+ const std::string ext = image.GetFileExtension();
+ Container container = Container::None;
+ if (ext == ".iso") {
+ container = Container::ISO;
+ } else if (ext == ".cso") {
+ container = Container::CSO;
+ } else if (ext == ".chd") {
+ container = Container::CHD;
+ } else if (ext == ".pbp") {
+ container = Container::PBP;
+ }
+ if (container != Container::None && file) {
+ *file = image;
+ }
+ return container;
+}
+
+std::vector
+
+ NotUsing
+
+
+ NotUsing
+ NotUsing
diff --git a/UWP/CoreUWP/CoreUWP.vcxproj.filters b/UWP/CoreUWP/CoreUWP.vcxproj.filters
index a6a3bde002..bef5fe63c4 100644
--- a/UWP/CoreUWP/CoreUWP.vcxproj.filters
+++ b/UWP/CoreUWP/CoreUWP.vcxproj.filters
@@ -62,6 +62,7 @@
+
@@ -397,6 +398,8 @@
+
+
@@ -472,6 +475,7 @@
+
@@ -681,6 +685,19 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/UWP/UI_UWP/UI_UWP.vcxproj b/UWP/UI_UWP/UI_UWP.vcxproj
index e24117cb7a..29566343cf 100644
--- a/UWP/UI_UWP/UI_UWP.vcxproj
+++ b/UWP/UI_UWP/UI_UWP.vcxproj
@@ -105,6 +105,7 @@
+
@@ -167,6 +168,7 @@
+
diff --git a/UWP/UI_UWP/UI_UWP.vcxproj.filters b/UWP/UI_UWP/UI_UWP.vcxproj.filters
index 020da9187b..3058a7df82 100644
--- a/UWP/UI_UWP/UI_UWP.vcxproj.filters
+++ b/UWP/UI_UWP/UI_UWP.vcxproj.filters
@@ -62,6 +62,9 @@
Screens
+
+ Screens
+ Screens
@@ -215,6 +218,9 @@
Screens
+
+ Screens
+ Screens
diff --git a/android/jni/Android.mk b/android/jni/Android.mk
index ead35d18bc..c0dd81c0b8 100644
--- a/android/jni/Android.mk
+++ b/android/jni/Android.mk
@@ -552,6 +552,8 @@ EXEC_AND_LIB_FILES := \
$(ARCH_FILES) \
$(GPU_VULKAN_FILES) \
$(SRC)/ext/xxhash.c \
+ $(SRC)/ext/libdeflate/lib/deflate_compress.c \
+ $(SRC)/ext/libdeflate/lib/utils.c \
$(SRC)/Core/MIPS/MIPS.cpp.arm \
$(SRC)/Core/MIPS/MIPSAnalyst.cpp \
$(SRC)/Core/MIPS/MIPSDis.cpp \
@@ -838,6 +840,7 @@ EXEC_AND_LIB_FILES := \
$(SRC)/Core/HLE/scePauth.cpp \
$(SRC)/Core/FileSystems/BlobFileSystem.cpp \
$(SRC)/Core/FileSystems/BlockDevices.cpp \
+ $(SRC)/Core/FileSystems/CSOCompress.cpp \
$(SRC)/Core/FileSystems/ISOFileSystem.cpp \
$(SRC)/Core/FileSystems/FileSystem.cpp \
$(SRC)/Core/FileSystems/MetaFileSystem.cpp \
@@ -1025,6 +1028,7 @@ LOCAL_SRC_FILES := \
$(SRC)/UI/LoadStateConfirmScreen.cpp \
$(SRC)/UI/GameInfoCache.cpp \
$(SRC)/UI/GameScreen.cpp \
+ $(SRC)/UI/GamePacker.cpp \
$(SRC)/UI/UploadScreen.cpp \
$(SRC)/UI/ControlMappingScreen.cpp \
$(SRC)/UI/GameSettingsScreen.cpp \
diff --git a/ext/CMakeLists.txt b/ext/CMakeLists.txt
index b8607ef950..d0d96f0eb8 100644
--- a/ext/CMakeLists.txt
+++ b/ext/CMakeLists.txt
@@ -51,6 +51,12 @@ add_library(xxhash STATIC
xxhash.h
)
+# Only the deflate encoder, for writing CSO files (Core/FileSystems/CSOCompress.cpp).
+add_library(libdeflate STATIC
+ libdeflate/lib/deflate_compress.c
+ libdeflate/lib/utils.c
+)
+
set(aemu_postoffice_client_obj
aemu_postoffice/client/postoffice.c
aemu_postoffice/client/postoffice_mem_stdc.c
diff --git a/ext/libdeflate/COPYING b/ext/libdeflate/COPYING
new file mode 100644
index 0000000000..1f1b81cd5a
--- /dev/null
+++ b/ext/libdeflate/COPYING
@@ -0,0 +1,21 @@
+Copyright 2016 Eric Biggers
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation files
+(the "Software"), to deal in the Software without restriction,
+including without limitation the rights to use, copy, modify, merge,
+publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/ext/libdeflate/README.ppsspp.md b/ext/libdeflate/README.ppsspp.md
new file mode 100644
index 0000000000..90979045d2
--- /dev/null
+++ b/ext/libdeflate/README.ppsspp.md
@@ -0,0 +1,6 @@
+libdeflate 1.7's deflate compressor (https://github.com/ebiggers/libdeflate), used to write CSO
+files in Core/FileSystems/CSOCompress.cpp. It's the version https://github.com/LunaMoo/maxcso
+vendors, so CSOs come out byte-identical to maxcso's. Only deflate_compress.c and utils.c and
+their headers are here. One change: lib/deflate_compress.c and lib/utils.c include
+"../libdeflate.h" instead of "libdeflate.h", so no extra include path is needed. MIT license,
+see COPYING.
diff --git a/ext/libdeflate/common/common_defs.h b/ext/libdeflate/common/common_defs.h
new file mode 100644
index 0000000000..d56c5cf9b4
--- /dev/null
+++ b/ext/libdeflate/common/common_defs.h
@@ -0,0 +1,334 @@
+/*
+ * common_defs.h
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef COMMON_COMMON_DEFS_H
+#define COMMON_COMMON_DEFS_H
+
+#ifdef __GNUC__
+# include "compiler_gcc.h"
+#elif defined(_MSC_VER)
+# include "compiler_msc.h"
+#else
+# pragma message("Unrecognized compiler. Please add a header file for your compiler. Compilation will proceed, but performance may suffer!")
+#endif
+
+/* ========================================================================== */
+/* Type definitions */
+/* ========================================================================== */
+
+#include /* size_t */
+
+#ifndef __bool_true_false_are_defined
+# include /* bool */
+#endif
+
+/* Fixed-width integer types */
+#include
+typedef uint8_t u8;
+typedef uint16_t u16;
+typedef uint32_t u32;
+typedef uint64_t u64;
+typedef int8_t s8;
+typedef int16_t s16;
+typedef int32_t s32;
+typedef int64_t s64;
+
+/*
+ * Word type of the target architecture. Use 'size_t' instead of 'unsigned
+ * long' to account for platforms such as Windows that use 32-bit 'unsigned
+ * long' on 64-bit architectures.
+ */
+typedef size_t machine_word_t;
+
+/* Number of bytes in a word */
+#define WORDBYTES ((int)sizeof(machine_word_t))
+
+/* Number of bits in a word */
+#define WORDBITS (8 * WORDBYTES)
+
+/* ========================================================================== */
+/* Optional compiler features */
+/* ========================================================================== */
+
+/* LIBEXPORT - export a function from a shared library */
+#ifndef LIBEXPORT
+# define LIBEXPORT
+#endif
+
+/* inline - suggest that a function be inlined */
+#ifndef inline
+# define inline
+#endif
+
+/* forceinline - force a function to be inlined, if possible */
+#ifndef forceinline
+# define forceinline inline
+#endif
+
+/* restrict - annotate a non-aliased pointer */
+#ifndef restrict
+# define restrict
+#endif
+
+/* likely(expr) - hint that an expression is usually true */
+#ifndef likely
+# define likely(expr) (expr)
+#endif
+
+/* unlikely(expr) - hint that an expression is usually false */
+#ifndef unlikely
+# define unlikely(expr) (expr)
+#endif
+
+/* prefetchr(addr) - prefetch into L1 cache for read */
+#ifndef prefetchr
+# define prefetchr(addr)
+#endif
+
+/* prefetchw(addr) - prefetch into L1 cache for write */
+#ifndef prefetchw
+# define prefetchw(addr)
+#endif
+
+/* Does the compiler support the 'target' function attribute? */
+#ifndef COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE
+# define COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE 0
+#endif
+
+/* Which targets are supported with the 'target' function attribute? */
+#ifndef COMPILER_SUPPORTS_BMI2_TARGET
+# define COMPILER_SUPPORTS_BMI2_TARGET 0
+#endif
+#ifndef COMPILER_SUPPORTS_AVX_TARGET
+# define COMPILER_SUPPORTS_AVX_TARGET 0
+#endif
+#ifndef COMPILER_SUPPORTS_AVX512BW_TARGET
+# define COMPILER_SUPPORTS_AVX512BW_TARGET 0
+#endif
+
+/*
+ * Which targets are supported with the 'target' function attribute and have
+ * intrinsics that work within 'target'-ed functions?
+ */
+#ifndef COMPILER_SUPPORTS_SSE2_TARGET_INTRINSICS
+# define COMPILER_SUPPORTS_SSE2_TARGET_INTRINSICS 0
+#endif
+#ifndef COMPILER_SUPPORTS_PCLMUL_TARGET_INTRINSICS
+# define COMPILER_SUPPORTS_PCLMUL_TARGET_INTRINSICS 0
+#endif
+#ifndef COMPILER_SUPPORTS_AVX2_TARGET_INTRINSICS
+# define COMPILER_SUPPORTS_AVX2_TARGET_INTRINSICS 0
+#endif
+#ifndef COMPILER_SUPPORTS_AVX512BW_TARGET_INTRINSICS
+# define COMPILER_SUPPORTS_AVX512BW_TARGET_INTRINSICS 0
+#endif
+#ifndef COMPILER_SUPPORTS_NEON_TARGET_INTRINSICS
+# define COMPILER_SUPPORTS_NEON_TARGET_INTRINSICS 0
+#endif
+#ifndef COMPILER_SUPPORTS_PMULL_TARGET_INTRINSICS
+# define COMPILER_SUPPORTS_PMULL_TARGET_INTRINSICS 0
+#endif
+#ifndef COMPILER_SUPPORTS_CRC32_TARGET_INTRINSICS
+# define COMPILER_SUPPORTS_CRC32_TARGET_INTRINSICS 0
+#endif
+
+/* _aligned_attribute(n) - declare that the annotated variable, or variables of
+ * the annotated type, are to be aligned on n-byte boundaries */
+#ifndef _aligned_attribute
+#endif
+
+/* ========================================================================== */
+/* Miscellaneous macros */
+/* ========================================================================== */
+
+#define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
+#define MIN(a, b) ((a) <= (b) ? (a) : (b))
+#define MAX(a, b) ((a) >= (b) ? (a) : (b))
+#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
+#define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
+#define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
+
+/* ========================================================================== */
+/* Endianness handling */
+/* ========================================================================== */
+
+/*
+ * CPU_IS_LITTLE_ENDIAN() - a macro which evaluates to 1 if the CPU is little
+ * endian or 0 if it is big endian. The macro should be defined in a way such
+ * that the compiler can evaluate it at compilation time. If not defined, a
+ * fallback is used.
+ */
+#ifndef CPU_IS_LITTLE_ENDIAN
+static forceinline int CPU_IS_LITTLE_ENDIAN(void)
+{
+ union {
+ unsigned int v;
+ unsigned char b;
+ } u;
+ u.v = 1;
+ return u.b;
+}
+#endif
+
+/* bswap16(n) - swap the bytes of a 16-bit integer */
+#ifndef bswap16
+static forceinline u16 bswap16(u16 n)
+{
+ return (n << 8) | (n >> 8);
+}
+#endif
+
+/* bswap32(n) - swap the bytes of a 32-bit integer */
+#ifndef bswap32
+static forceinline u32 bswap32(u32 n)
+{
+ return ((n & 0x000000FF) << 24) |
+ ((n & 0x0000FF00) << 8) |
+ ((n & 0x00FF0000) >> 8) |
+ ((n & 0xFF000000) >> 24);
+}
+#endif
+
+/* bswap64(n) - swap the bytes of a 64-bit integer */
+#ifndef bswap64
+static forceinline u64 bswap64(u64 n)
+{
+ return ((n & 0x00000000000000FF) << 56) |
+ ((n & 0x000000000000FF00) << 40) |
+ ((n & 0x0000000000FF0000) << 24) |
+ ((n & 0x00000000FF000000) << 8) |
+ ((n & 0x000000FF00000000) >> 8) |
+ ((n & 0x0000FF0000000000) >> 24) |
+ ((n & 0x00FF000000000000) >> 40) |
+ ((n & 0xFF00000000000000) >> 56);
+}
+#endif
+
+#define le16_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? (n) : bswap16(n))
+#define le32_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? (n) : bswap32(n))
+#define le64_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? (n) : bswap64(n))
+#define be16_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? bswap16(n) : (n))
+#define be32_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? bswap32(n) : (n))
+#define be64_bswap(n) (CPU_IS_LITTLE_ENDIAN() ? bswap64(n) : (n))
+
+/* ========================================================================== */
+/* Unaligned memory accesses */
+/* ========================================================================== */
+
+/*
+ * UNALIGNED_ACCESS_IS_FAST should be defined to 1 if unaligned memory accesses
+ * can be performed efficiently on the target platform.
+ */
+#ifndef UNALIGNED_ACCESS_IS_FAST
+# define UNALIGNED_ACCESS_IS_FAST 0
+#endif
+
+/* ========================================================================== */
+/* Bit scan functions */
+/* ========================================================================== */
+
+/*
+ * Bit Scan Reverse (BSR) - find the 0-based index (relative to the least
+ * significant end) of the *most* significant 1 bit in the input value. The
+ * input value must be nonzero!
+ */
+
+#ifndef bsr32
+static forceinline unsigned
+bsr32(u32 n)
+{
+ unsigned i = 0;
+ while ((n >>= 1) != 0)
+ i++;
+ return i;
+}
+#endif
+
+#ifndef bsr64
+static forceinline unsigned
+bsr64(u64 n)
+{
+ unsigned i = 0;
+ while ((n >>= 1) != 0)
+ i++;
+ return i;
+}
+#endif
+
+static forceinline unsigned
+bsrw(machine_word_t n)
+{
+ STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
+ if (WORDBITS == 32)
+ return bsr32(n);
+ else
+ return bsr64(n);
+}
+
+/*
+ * Bit Scan Forward (BSF) - find the 0-based index (relative to the least
+ * significant end) of the *least* significant 1 bit in the input value. The
+ * input value must be nonzero!
+ */
+
+#ifndef bsf32
+static forceinline unsigned
+bsf32(u32 n)
+{
+ unsigned i = 0;
+ while ((n & 1) == 0) {
+ i++;
+ n >>= 1;
+ }
+ return i;
+}
+#endif
+
+#ifndef bsf64
+static forceinline unsigned
+bsf64(u64 n)
+{
+ unsigned i = 0;
+ while ((n & 1) == 0) {
+ i++;
+ n >>= 1;
+ }
+ return i;
+}
+#endif
+
+static forceinline unsigned
+bsfw(machine_word_t n)
+{
+ STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
+ if (WORDBITS == 32)
+ return bsf32(n);
+ else
+ return bsf64(n);
+}
+
+#endif /* COMMON_COMMON_DEFS_H */
diff --git a/ext/libdeflate/common/compiler_gcc.h b/ext/libdeflate/common/compiler_gcc.h
new file mode 100644
index 0000000000..82599469a6
--- /dev/null
+++ b/ext/libdeflate/common/compiler_gcc.h
@@ -0,0 +1,185 @@
+/*
+ * compiler_gcc.h - definitions for the GNU C Compiler. This also handles clang
+ * and the Intel C Compiler (icc).
+ *
+ * TODO: icc is not well tested, so some things are currently disabled even
+ * though they maybe can be enabled on some icc versions.
+ */
+
+#if !defined(__clang__) && !defined(__INTEL_COMPILER)
+# define GCC_PREREQ(major, minor) \
+ (__GNUC__ > (major) || \
+ (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
+#else
+# define GCC_PREREQ(major, minor) 0
+#endif
+
+/* Note: only check the clang version when absolutely necessary!
+ * "Vendors" such as Apple can use different version numbers. */
+#ifdef __clang__
+# ifdef __apple_build_version__
+# define CLANG_PREREQ(major, minor, apple_version) \
+ (__apple_build_version__ >= (apple_version))
+# else
+# define CLANG_PREREQ(major, minor, apple_version) \
+ (__clang_major__ > (major) || \
+ (__clang_major__ == (major) && __clang_minor__ >= (minor)))
+# endif
+#else
+# define CLANG_PREREQ(major, minor, apple_version) 0
+#endif
+
+#ifndef __has_attribute
+# define __has_attribute(attribute) 0
+#endif
+#ifndef __has_feature
+# define __has_feature(feature) 0
+#endif
+#ifndef __has_builtin
+# define __has_builtin(builtin) 0
+#endif
+
+#ifdef _WIN32
+# define LIBEXPORT __declspec(dllexport)
+#else
+# define LIBEXPORT __attribute__((visibility("default")))
+#endif
+
+#define inline inline
+#define forceinline inline __attribute__((always_inline))
+#define restrict __restrict__
+#define likely(expr) __builtin_expect(!!(expr), 1)
+#define unlikely(expr) __builtin_expect(!!(expr), 0)
+#define prefetchr(addr) __builtin_prefetch((addr), 0)
+#define prefetchw(addr) __builtin_prefetch((addr), 1)
+#define _aligned_attribute(n) __attribute__((aligned(n)))
+
+#define COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE \
+ (GCC_PREREQ(4, 4) || __has_attribute(target))
+
+#if COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE
+
+# if defined(__i386__) || defined(__x86_64__)
+
+# define COMPILER_SUPPORTS_PCLMUL_TARGET \
+ (GCC_PREREQ(4, 4) || __has_builtin(__builtin_ia32_pclmulqdq128))
+
+# define COMPILER_SUPPORTS_AVX_TARGET \
+ (GCC_PREREQ(4, 6) || __has_builtin(__builtin_ia32_maxps256))
+
+# define COMPILER_SUPPORTS_BMI2_TARGET \
+ (GCC_PREREQ(4, 7) || __has_builtin(__builtin_ia32_pdep_di))
+
+# define COMPILER_SUPPORTS_AVX2_TARGET \
+ (GCC_PREREQ(4, 7) || __has_builtin(__builtin_ia32_psadbw256))
+
+# define COMPILER_SUPPORTS_AVX512BW_TARGET \
+ (GCC_PREREQ(5, 1) || __has_builtin(__builtin_ia32_psadbw512))
+
+ /*
+ * Prior to gcc 4.9 (r200349) and clang 3.8 (r239883), x86 intrinsics
+ * not available in the main target could not be used in 'target'
+ * attribute functions. Unfortunately clang has no feature test macro
+ * for this so we have to check its version.
+ */
+# if GCC_PREREQ(4, 9) || CLANG_PREREQ(3, 8, 7030000)
+# define COMPILER_SUPPORTS_SSE2_TARGET_INTRINSICS 1
+# define COMPILER_SUPPORTS_PCLMUL_TARGET_INTRINSICS \
+ COMPILER_SUPPORTS_PCLMUL_TARGET
+# define COMPILER_SUPPORTS_AVX2_TARGET_INTRINSICS \
+ COMPILER_SUPPORTS_AVX2_TARGET
+# define COMPILER_SUPPORTS_AVX512BW_TARGET_INTRINSICS \
+ COMPILER_SUPPORTS_AVX512BW_TARGET
+# endif
+
+# elif defined(__arm__) || defined(__aarch64__)
+
+ /*
+ * Determine whether NEON and crypto intrinsics are supported.
+ *
+ * With gcc prior to 6.1, (r230411 for arm32, r226563 for arm64), neither
+ * was available unless enabled in the main target.
+ *
+ * But even after that, to include (which contains both the
+ * basic NEON intrinsics and the crypto intrinsics) the main target still
+ * needs to have:
+ * - gcc: hardware floating point support
+ * - clang: NEON support (but not necessarily crypto support)
+ */
+# if (GCC_PREREQ(6, 1) && defined(__ARM_FP)) || \
+ (defined(__clang__) && defined(__ARM_NEON))
+# define COMPILER_SUPPORTS_NEON_TARGET_INTRINSICS 1
+ /*
+ * The crypto intrinsics are broken on arm32 with clang, even when using
+ * -mfpu=crypto-neon-fp-armv8, because clang's puts them
+ * behind __aarch64__. Undefine __ARM_FEATURE_CRYPTO in that case...
+ */
+# if defined(__clang__) && defined(__arm__)
+# undef __ARM_FEATURE_CRYPTO
+# elif __has_builtin(__builtin_neon_vmull_p64) || !defined(__clang__)
+# define COMPILER_SUPPORTS_PMULL_TARGET_INTRINSICS 1
+# endif
+# endif
+
+ /*
+ * Determine whether CRC32 intrinsics are supported.
+ *
+ * With gcc r274827 or later (gcc 10.1+, 9.3+, or 8.4+), or with clang,
+ * they work as expected. (Well, not quite. There's still a bug, but we
+ * have to work around it later when including arm_acle.h.)
+ */
+# if GCC_PREREQ(10, 1) || \
+ (GCC_PREREQ(9, 3) && !GCC_PREREQ(10, 0)) || \
+ (GCC_PREREQ(8, 4) && !GCC_PREREQ(9, 0)) || \
+ (defined(__clang__) && __has_builtin(__builtin_arm_crc32b))
+# define COMPILER_SUPPORTS_CRC32_TARGET_INTRINSICS 1
+# endif
+
+# endif /* __arm__ || __aarch64__ */
+
+#endif /* COMPILER_SUPPORTS_TARGET_FUNCTION_ATTRIBUTE */
+
+/*
+ * Prior to gcc 5.1 and clang 3.9, emmintrin.h only defined vectors of signed
+ * integers (e.g. __v4si), not vectors of unsigned integers (e.g. __v4su). But
+ * we need the unsigned ones in order to avoid signed integer overflow, which is
+ * undefined behavior. Add the missing definitions for the unsigned ones if
+ * needed.
+ */
+#if (GCC_PREREQ(4, 0) && !GCC_PREREQ(5, 1)) || \
+ (defined(__clang__) && !CLANG_PREREQ(3, 9, 8020000))
+typedef unsigned long long __v2du __attribute__((__vector_size__(16)));
+typedef unsigned int __v4su __attribute__((__vector_size__(16)));
+typedef unsigned short __v8hu __attribute__((__vector_size__(16)));
+typedef unsigned char __v16qu __attribute__((__vector_size__(16)));
+typedef unsigned long long __v4du __attribute__((__vector_size__(32)));
+typedef unsigned int __v8su __attribute__((__vector_size__(32)));
+typedef unsigned short __v16hu __attribute__((__vector_size__(32)));
+typedef unsigned char __v32qu __attribute__((__vector_size__(32)));
+#endif
+
+/* Newer gcc supports __BYTE_ORDER__. Older gcc doesn't. */
+#ifdef __BYTE_ORDER__
+# define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+#endif
+
+#if GCC_PREREQ(4, 8) || __has_builtin(__builtin_bswap16)
+# define bswap16 __builtin_bswap16
+#endif
+
+#if GCC_PREREQ(4, 3) || __has_builtin(__builtin_bswap32)
+# define bswap32 __builtin_bswap32
+#endif
+
+#if GCC_PREREQ(4, 3) || __has_builtin(__builtin_bswap64)
+# define bswap64 __builtin_bswap64
+#endif
+
+#if defined(__x86_64__) || defined(__i386__) || defined(__ARM_FEATURE_UNALIGNED) || defined(__powerpc64__)
+# define UNALIGNED_ACCESS_IS_FAST 1
+#endif
+
+#define bsr32(n) (31 - __builtin_clz(n))
+#define bsr64(n) (63 - __builtin_clzll(n))
+#define bsf32(n) __builtin_ctz(n)
+#define bsf64(n) __builtin_ctzll(n)
diff --git a/ext/libdeflate/common/compiler_msc.h b/ext/libdeflate/common/compiler_msc.h
new file mode 100644
index 0000000000..18cfa128f1
--- /dev/null
+++ b/ext/libdeflate/common/compiler_msc.h
@@ -0,0 +1,80 @@
+/*
+ * compiler_msc.h - definitions for the Microsoft C Compiler
+ */
+
+#include
+#include /* for _byteswap_*() */
+
+#define LIBEXPORT __declspec(dllexport)
+
+/*
+ * Old versions (e.g. VS2010) of MSC don't have the C99 header stdbool.h.
+ * Beware: the below replacement isn't fully standard, since normally any value
+ * != 0 should be implicitly cast to a bool with value 1... but that doesn't
+ * happen if bool is really just an 'int'.
+ */
+typedef int bool;
+#define true 1
+#define false 0
+#define __bool_true_false_are_defined 1
+
+/* Define ssize_t */
+#ifdef _WIN64
+typedef long long ssize_t;
+#else
+typedef int ssize_t;
+#endif
+
+/* Assume a little endian architecture with fast unaligned access */
+#define CPU_IS_LITTLE_ENDIAN() 1
+#define UNALIGNED_ACCESS_IS_FAST 1
+
+/* __restrict has nonstandard behavior; don't use it */
+#define restrict
+
+/* ... but we can use __inline and __forceinline */
+#define inline __inline
+#define forceinline __forceinline
+
+/* Byte swap functions */
+#define bswap16 _byteswap_ushort
+#define bswap32 _byteswap_ulong
+#define bswap64 _byteswap_uint64
+
+/* Bit scan functions (32-bit) */
+
+static forceinline unsigned
+bsr32(uint32_t n)
+{
+ _BitScanReverse(&n, n);
+ return n;
+}
+#define bsr32 bsr32
+
+static forceinline unsigned
+bsf32(uint32_t n)
+{
+ _BitScanForward(&n, n);
+ return n;
+}
+#define bsf32 bsf32
+
+#ifdef _M_X64 /* Bit scan functions (64-bit) */
+
+static forceinline unsigned
+bsr64(uint64_t n)
+{
+ _BitScanReverse64(&n, n);
+ return n;
+}
+#define bsr64 bsr64
+
+static forceinline unsigned
+bsf64(uint64_t n)
+{
+ _BitScanForward64(&n, n);
+ return n;
+}
+#define bsf64 bsf64
+
+#endif /* _M_X64 */
diff --git a/ext/libdeflate/lib/arm/matchfinder_impl.h b/ext/libdeflate/lib/arm/matchfinder_impl.h
new file mode 100644
index 0000000000..9e711cc646
--- /dev/null
+++ b/ext/libdeflate/lib/arm/matchfinder_impl.h
@@ -0,0 +1,81 @@
+/*
+ * arm/matchfinder_impl.h - ARM implementations of matchfinder functions
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifdef __ARM_NEON
+# include
+static forceinline void
+matchfinder_init_neon(mf_pos_t *data, size_t size)
+{
+ int16x8_t *p = (int16x8_t *)data;
+ int16x8_t v = (int16x8_t) {
+ MATCHFINDER_INITVAL, MATCHFINDER_INITVAL, MATCHFINDER_INITVAL,
+ MATCHFINDER_INITVAL, MATCHFINDER_INITVAL, MATCHFINDER_INITVAL,
+ MATCHFINDER_INITVAL, MATCHFINDER_INITVAL,
+ };
+
+ STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
+ STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
+ STATIC_ASSERT(sizeof(mf_pos_t) == 2);
+
+ do {
+ p[0] = v;
+ p[1] = v;
+ p[2] = v;
+ p[3] = v;
+ p += 4;
+ size -= 4 * sizeof(*p);
+ } while (size != 0);
+}
+#define matchfinder_init matchfinder_init_neon
+
+static forceinline void
+matchfinder_rebase_neon(mf_pos_t *data, size_t size)
+{
+ int16x8_t *p = (int16x8_t *)data;
+ int16x8_t v = (int16x8_t) {
+ (u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
+ (u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
+ (u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
+ (u16)-MATCHFINDER_WINDOW_SIZE, (u16)-MATCHFINDER_WINDOW_SIZE,
+ };
+
+ STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
+ STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
+ STATIC_ASSERT(sizeof(mf_pos_t) == 2);
+
+ do {
+ p[0] = vqaddq_s16(p[0], v);
+ p[1] = vqaddq_s16(p[1], v);
+ p[2] = vqaddq_s16(p[2], v);
+ p[3] = vqaddq_s16(p[3], v);
+ p += 4;
+ size -= 4 * sizeof(*p);
+ } while (size != 0);
+}
+#define matchfinder_rebase matchfinder_rebase_neon
+
+#endif /* __ARM_NEON */
diff --git a/ext/libdeflate/lib/bt_matchfinder.h b/ext/libdeflate/lib/bt_matchfinder.h
new file mode 100644
index 0000000000..546a19d31a
--- /dev/null
+++ b/ext/libdeflate/lib/bt_matchfinder.h
@@ -0,0 +1,359 @@
+/*
+ * bt_matchfinder.h - Lempel-Ziv matchfinding with a hash table of binary trees
+ *
+ * Originally public domain; changes after 2016-09-07 are copyrighted.
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * ----------------------------------------------------------------------------
+ *
+ * This is a Binary Trees (bt) based matchfinder.
+ *
+ * The main data structure is a hash table where each hash bucket contains a
+ * binary tree of sequences whose first 4 bytes share the same hash code. Each
+ * sequence is identified by its starting position in the input buffer. Each
+ * binary tree is always sorted such that each left child represents a sequence
+ * lexicographically lesser than its parent and each right child represents a
+ * sequence lexicographically greater than its parent.
+ *
+ * The algorithm processes the input buffer sequentially. At each byte
+ * position, the hash code of the first 4 bytes of the sequence beginning at
+ * that position (the sequence being matched against) is computed. This
+ * identifies the hash bucket to use for that position. Then, a new binary tree
+ * node is created to represent the current sequence. Then, in a single tree
+ * traversal, the hash bucket's binary tree is searched for matches and is
+ * re-rooted at the new node.
+ *
+ * Compared to the simpler algorithm that uses linked lists instead of binary
+ * trees (see hc_matchfinder.h), the binary tree version gains more information
+ * at each node visitation. Ideally, the binary tree version will examine only
+ * 'log(n)' nodes to find the same matches that the linked list version will
+ * find by examining 'n' nodes. In addition, the binary tree version can
+ * examine fewer bytes at each node by taking advantage of the common prefixes
+ * that result from the sort order, whereas the linked list version may have to
+ * examine up to the full length of the match at each node.
+ *
+ * However, it is not always best to use the binary tree version. It requires
+ * nearly twice as much memory as the linked list version, and it takes time to
+ * keep the binary trees sorted, even at positions where the compressor does not
+ * need matches. Generally, when doing fast compression on small buffers,
+ * binary trees are the wrong approach. They are best suited for thorough
+ * compression and/or large buffers.
+ *
+ * ----------------------------------------------------------------------------
+ */
+
+
+#include "matchfinder_common.h"
+
+#define BT_MATCHFINDER_HASH3_ORDER 16
+#define BT_MATCHFINDER_HASH3_WAYS 2
+#define BT_MATCHFINDER_HASH4_ORDER 16
+
+#define BT_MATCHFINDER_TOTAL_HASH_SIZE \
+ (((1UL << BT_MATCHFINDER_HASH3_ORDER) * BT_MATCHFINDER_HASH3_WAYS + \
+ (1UL << BT_MATCHFINDER_HASH4_ORDER)) * sizeof(mf_pos_t))
+
+/* Representation of a match found by the bt_matchfinder */
+struct lz_match {
+
+ /* The number of bytes matched. */
+ u16 length;
+
+ /* The offset back from the current position that was matched. */
+ u16 offset;
+};
+
+struct bt_matchfinder {
+
+ /* The hash table for finding length 3 matches */
+ mf_pos_t hash3_tab[1UL << BT_MATCHFINDER_HASH3_ORDER][BT_MATCHFINDER_HASH3_WAYS];
+
+ /* The hash table which contains the roots of the binary trees for
+ * finding length 4+ matches */
+ mf_pos_t hash4_tab[1UL << BT_MATCHFINDER_HASH4_ORDER];
+
+ /* The child node references for the binary trees. The left and right
+ * children of the node for the sequence with position 'pos' are
+ * 'child_tab[pos * 2]' and 'child_tab[pos * 2 + 1]', respectively. */
+ mf_pos_t child_tab[2UL * MATCHFINDER_WINDOW_SIZE];
+
+}
+#ifdef _aligned_attribute
+_aligned_attribute(MATCHFINDER_MEM_ALIGNMENT)
+#endif
+;
+
+/* Prepare the matchfinder for a new input buffer. */
+static forceinline void
+bt_matchfinder_init(struct bt_matchfinder *mf)
+{
+ STATIC_ASSERT(BT_MATCHFINDER_TOTAL_HASH_SIZE %
+ MATCHFINDER_SIZE_ALIGNMENT == 0);
+
+ matchfinder_init((mf_pos_t *)mf, BT_MATCHFINDER_TOTAL_HASH_SIZE);
+}
+
+static forceinline void
+bt_matchfinder_slide_window(struct bt_matchfinder *mf)
+{
+ STATIC_ASSERT(sizeof(*mf) % MATCHFINDER_SIZE_ALIGNMENT == 0);
+
+ matchfinder_rebase((mf_pos_t *)mf, sizeof(*mf));
+}
+
+static forceinline mf_pos_t *
+bt_left_child(struct bt_matchfinder *mf, s32 node)
+{
+ return &mf->child_tab[2 * (node & (MATCHFINDER_WINDOW_SIZE - 1)) + 0];
+}
+
+static forceinline mf_pos_t *
+bt_right_child(struct bt_matchfinder *mf, s32 node)
+{
+ return &mf->child_tab[2 * (node & (MATCHFINDER_WINDOW_SIZE - 1)) + 1];
+}
+
+/* The minimum permissible value of 'max_len' for bt_matchfinder_get_matches()
+ * and bt_matchfinder_skip_position(). There must be sufficiently many bytes
+ * remaining to load a 32-bit integer from the *next* position. */
+#define BT_MATCHFINDER_REQUIRED_NBYTES 5
+
+/* Advance the binary tree matchfinder by one byte, optionally recording
+ * matches. @record_matches should be a compile-time constant. */
+static forceinline struct lz_match *
+bt_matchfinder_advance_one_byte(struct bt_matchfinder * const restrict mf,
+ const u8 * const restrict in_base,
+ const ptrdiff_t cur_pos,
+ const u32 max_len,
+ const u32 nice_len,
+ const u32 max_search_depth,
+ u32 * const restrict next_hashes,
+ u32 * const restrict best_len_ret,
+ struct lz_match * restrict lz_matchptr,
+ const bool record_matches)
+{
+ const u8 *in_next = in_base + cur_pos;
+ u32 depth_remaining = max_search_depth;
+ const s32 cutoff = cur_pos - MATCHFINDER_WINDOW_SIZE;
+ u32 next_hashseq;
+ u32 hash3;
+ u32 hash4;
+ s32 cur_node;
+#if BT_MATCHFINDER_HASH3_WAYS >= 2
+ s32 cur_node_2;
+#endif
+ const u8 *matchptr;
+ mf_pos_t *pending_lt_ptr, *pending_gt_ptr;
+ u32 best_lt_len, best_gt_len;
+ u32 len;
+ u32 best_len = 3;
+
+ STATIC_ASSERT(BT_MATCHFINDER_HASH3_WAYS >= 1 &&
+ BT_MATCHFINDER_HASH3_WAYS <= 2);
+
+ next_hashseq = get_unaligned_le32(in_next + 1);
+
+ hash3 = next_hashes[0];
+ hash4 = next_hashes[1];
+
+ next_hashes[0] = lz_hash(next_hashseq & 0xFFFFFF, BT_MATCHFINDER_HASH3_ORDER);
+ next_hashes[1] = lz_hash(next_hashseq, BT_MATCHFINDER_HASH4_ORDER);
+ prefetchw(&mf->hash3_tab[next_hashes[0]]);
+ prefetchw(&mf->hash4_tab[next_hashes[1]]);
+
+ cur_node = mf->hash3_tab[hash3][0];
+ mf->hash3_tab[hash3][0] = cur_pos;
+#if BT_MATCHFINDER_HASH3_WAYS >= 2
+ cur_node_2 = mf->hash3_tab[hash3][1];
+ mf->hash3_tab[hash3][1] = cur_node;
+#endif
+ if (record_matches && cur_node > cutoff) {
+ u32 seq3 = load_u24_unaligned(in_next);
+ if (seq3 == load_u24_unaligned(&in_base[cur_node])) {
+ lz_matchptr->length = 3;
+ lz_matchptr->offset = in_next - &in_base[cur_node];
+ lz_matchptr++;
+ }
+ #if BT_MATCHFINDER_HASH3_WAYS >= 2
+ else if (cur_node_2 > cutoff &&
+ seq3 == load_u24_unaligned(&in_base[cur_node_2]))
+ {
+ lz_matchptr->length = 3;
+ lz_matchptr->offset = in_next - &in_base[cur_node_2];
+ lz_matchptr++;
+ }
+ #endif
+ }
+
+ cur_node = mf->hash4_tab[hash4];
+ mf->hash4_tab[hash4] = cur_pos;
+
+ pending_lt_ptr = bt_left_child(mf, cur_pos);
+ pending_gt_ptr = bt_right_child(mf, cur_pos);
+
+ if (cur_node <= cutoff) {
+ *pending_lt_ptr = MATCHFINDER_INITVAL;
+ *pending_gt_ptr = MATCHFINDER_INITVAL;
+ *best_len_ret = best_len;
+ return lz_matchptr;
+ }
+
+ best_lt_len = 0;
+ best_gt_len = 0;
+ len = 0;
+
+ for (;;) {
+ matchptr = &in_base[cur_node];
+
+ if (matchptr[len] == in_next[len]) {
+ len = lz_extend(in_next, matchptr, len + 1, max_len);
+ if (!record_matches || len > best_len) {
+ if (record_matches) {
+ best_len = len;
+ lz_matchptr->length = len;
+ lz_matchptr->offset = in_next - matchptr;
+ lz_matchptr++;
+ }
+ if (len >= nice_len) {
+ *pending_lt_ptr = *bt_left_child(mf, cur_node);
+ *pending_gt_ptr = *bt_right_child(mf, cur_node);
+ *best_len_ret = best_len;
+ return lz_matchptr;
+ }
+ }
+ }
+
+ if (matchptr[len] < in_next[len]) {
+ *pending_lt_ptr = cur_node;
+ pending_lt_ptr = bt_right_child(mf, cur_node);
+ cur_node = *pending_lt_ptr;
+ best_lt_len = len;
+ if (best_gt_len < len)
+ len = best_gt_len;
+ } else {
+ *pending_gt_ptr = cur_node;
+ pending_gt_ptr = bt_left_child(mf, cur_node);
+ cur_node = *pending_gt_ptr;
+ best_gt_len = len;
+ if (best_lt_len < len)
+ len = best_lt_len;
+ }
+
+ if (cur_node <= cutoff || !--depth_remaining) {
+ *pending_lt_ptr = MATCHFINDER_INITVAL;
+ *pending_gt_ptr = MATCHFINDER_INITVAL;
+ *best_len_ret = best_len;
+ return lz_matchptr;
+ }
+ }
+}
+
+/*
+ * Retrieve a list of matches with the current position.
+ *
+ * @mf
+ * The matchfinder structure.
+ * @in_base
+ * Pointer to the next byte in the input buffer to process _at the last
+ * time bt_matchfinder_init() or bt_matchfinder_slide_window() was called_.
+ * @cur_pos
+ * The current position in the input buffer relative to @in_base (the
+ * position of the sequence being matched against).
+ * @max_len
+ * The maximum permissible match length at this position. Must be >=
+ * BT_MATCHFINDER_REQUIRED_NBYTES.
+ * @nice_len
+ * Stop searching if a match of at least this length is found.
+ * Must be <= @max_len.
+ * @max_search_depth
+ * Limit on the number of potential matches to consider. Must be >= 1.
+ * @next_hashes
+ * The precomputed hash codes for the sequence beginning at @in_next.
+ * These will be used and then updated with the precomputed hashcodes for
+ * the sequence beginning at @in_next + 1.
+ * @best_len_ret
+ * If a match of length >= 4 was found, then the length of the longest such
+ * match is written here; otherwise 3 is written here. (Note: this is
+ * redundant with the 'struct lz_match' array, but this is easier for the
+ * compiler to optimize when inlined and the caller immediately does a
+ * check against 'best_len'.)
+ * @lz_matchptr
+ * An array in which this function will record the matches. The recorded
+ * matches will be sorted by strictly increasing length and (non-strictly)
+ * increasing offset. The maximum number of matches that may be found is
+ * 'nice_len - 2'.
+ *
+ * The return value is a pointer to the next available slot in the @lz_matchptr
+ * array. (If no matches were found, this will be the same as @lz_matchptr.)
+ */
+static forceinline struct lz_match *
+bt_matchfinder_get_matches(struct bt_matchfinder *mf,
+ const u8 *in_base,
+ ptrdiff_t cur_pos,
+ u32 max_len,
+ u32 nice_len,
+ u32 max_search_depth,
+ u32 next_hashes[2],
+ u32 *best_len_ret,
+ struct lz_match *lz_matchptr)
+{
+ return bt_matchfinder_advance_one_byte(mf,
+ in_base,
+ cur_pos,
+ max_len,
+ nice_len,
+ max_search_depth,
+ next_hashes,
+ best_len_ret,
+ lz_matchptr,
+ true);
+}
+
+/*
+ * Advance the matchfinder, but don't record any matches.
+ *
+ * This is very similar to bt_matchfinder_get_matches() because both functions
+ * must do hashing and tree re-rooting.
+ */
+static forceinline void
+bt_matchfinder_skip_position(struct bt_matchfinder *mf,
+ const u8 *in_base,
+ ptrdiff_t cur_pos,
+ u32 nice_len,
+ u32 max_search_depth,
+ u32 next_hashes[2])
+{
+ u32 best_len;
+ bt_matchfinder_advance_one_byte(mf,
+ in_base,
+ cur_pos,
+ nice_len,
+ nice_len,
+ max_search_depth,
+ next_hashes,
+ &best_len,
+ NULL,
+ false);
+}
diff --git a/ext/libdeflate/lib/deflate_compress.c b/ext/libdeflate/lib/deflate_compress.c
new file mode 100644
index 0000000000..eb1d614d3d
--- /dev/null
+++ b/ext/libdeflate/lib/deflate_compress.c
@@ -0,0 +1,2854 @@
+/*
+ * deflate_compress.c - a compressor for DEFLATE
+ *
+ * Originally public domain; changes after 2016-09-07 are copyrighted.
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "deflate_compress.h"
+#include "deflate_constants.h"
+#include "unaligned.h"
+
+#include "../libdeflate.h"
+
+/*
+ * By default, the near-optimal parsing algorithm is enabled at compression
+ * level 8 and above. The near-optimal parsing algorithm produces a compression
+ * ratio significantly better than the greedy and lazy algorithms implemented
+ * here, and also the algorithm used by zlib at level 9. However, it is slow.
+ */
+#define SUPPORT_NEAR_OPTIMAL_PARSING 1
+
+/*
+ * Define to 1 to maintain the full map from match offsets to offset slots.
+ * This slightly speeds up translations of match offsets to offset slots, but it
+ * uses 32769 bytes of memory rather than the 512 bytes used by the condensed
+ * map. The speedup provided by the larger map is most helpful when the
+ * near-optimal parsing algorithm is being used.
+ */
+#define USE_FULL_OFFSET_SLOT_FAST SUPPORT_NEAR_OPTIMAL_PARSING
+
+/*
+ * DEFLATE uses a 32768 byte sliding window; set the matchfinder parameters
+ * appropriately.
+ */
+#define MATCHFINDER_WINDOW_ORDER 15
+
+#include "hc_matchfinder.h"
+#if SUPPORT_NEAR_OPTIMAL_PARSING
+# include "bt_matchfinder.h"
+#endif
+
+/*
+ * The compressor always chooses a block of at least MIN_BLOCK_LENGTH bytes,
+ * except if the last block has to be shorter.
+ */
+#define MIN_BLOCK_LENGTH 10000
+
+/*
+ * The compressor attempts to end blocks after SOFT_MAX_BLOCK_LENGTH bytes, but
+ * the final length might be slightly longer due to matches extending beyond
+ * this limit.
+ */
+#define SOFT_MAX_BLOCK_LENGTH 300000
+
+/*
+ * The number of observed matches or literals that represents sufficient data to
+ * decide whether the current block should be terminated or not.
+ */
+#define NUM_OBSERVATIONS_PER_BLOCK_CHECK 512
+
+
+#if SUPPORT_NEAR_OPTIMAL_PARSING
+/* Constants specific to the near-optimal parsing algorithm */
+
+/*
+ * The maximum number of matches the matchfinder can find at a single position.
+ * Since the matchfinder never finds more than one match for the same length,
+ * presuming one of each possible length is sufficient for an upper bound.
+ * (This says nothing about whether it is worthwhile to consider so many
+ * matches; this is just defining the worst case.)
+ */
+# define MAX_MATCHES_PER_POS (DEFLATE_MAX_MATCH_LEN - DEFLATE_MIN_MATCH_LEN + 1)
+
+/*
+ * The number of lz_match structures in the match cache, excluding the extra
+ * "overflow" entries. This value should be high enough so that nearly the
+ * time, all matches found in a given block can fit in the match cache.
+ * However, fallback behavior (immediately terminating the block) on cache
+ * overflow is still required.
+ */
+# define CACHE_LENGTH (SOFT_MAX_BLOCK_LENGTH * 5)
+
+#endif /* SUPPORT_NEAR_OPTIMAL_PARSING */
+
+/*
+ * These are the compressor-side limits on the codeword lengths for each Huffman
+ * code. To make outputting bits slightly faster, some of these limits are
+ * lower than the limits defined by the DEFLATE format. This does not
+ * significantly affect the compression ratio, at least for the block lengths we
+ * use.
+ */
+#define MAX_LITLEN_CODEWORD_LEN 14
+#define MAX_OFFSET_CODEWORD_LEN DEFLATE_MAX_OFFSET_CODEWORD_LEN
+#define MAX_PRE_CODEWORD_LEN DEFLATE_MAX_PRE_CODEWORD_LEN
+
+/* Table: length slot => length slot base value */
+static const unsigned deflate_length_slot_base[] = {
+ 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 ,
+ 11 , 13 , 15 , 17 , 19 , 23 , 27 , 31 ,
+ 35 , 43 , 51 , 59 , 67 , 83 , 99 , 115 ,
+ 131 , 163 , 195 , 227 , 258 ,
+};
+
+/* Table: length slot => number of extra length bits */
+static const u8 deflate_extra_length_bits[] = {
+ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
+ 1 , 1 , 1 , 1 , 2 , 2 , 2 , 2 ,
+ 3 , 3 , 3 , 3 , 4 , 4 , 4 , 4 ,
+ 5 , 5 , 5 , 5 , 0 ,
+};
+
+/* Table: offset slot => offset slot base value */
+static const unsigned deflate_offset_slot_base[] = {
+ 1 , 2 , 3 , 4 , 5 , 7 , 9 , 13 ,
+ 17 , 25 , 33 , 49 , 65 , 97 , 129 , 193 ,
+ 257 , 385 , 513 , 769 , 1025 , 1537 , 2049 , 3073 ,
+ 4097 , 6145 , 8193 , 12289 , 16385 , 24577 ,
+};
+
+/* Table: offset slot => number of extra offset bits */
+static const u8 deflate_extra_offset_bits[] = {
+ 0 , 0 , 0 , 0 , 1 , 1 , 2 , 2 ,
+ 3 , 3 , 4 , 4 , 5 , 5 , 6 , 6 ,
+ 7 , 7 , 8 , 8 , 9 , 9 , 10 , 10 ,
+ 11 , 11 , 12 , 12 , 13 , 13 ,
+};
+
+/* Table: length => length slot */
+static const u8 deflate_length_slot[DEFLATE_MAX_MATCH_LEN + 1] = {
+ 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 12,
+ 12, 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, 15, 15, 16, 16, 16, 16, 16,
+ 16, 16, 16, 17, 17, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 18, 18,
+ 18, 19, 19, 19, 19, 19, 19, 19, 19, 20, 20, 20, 20, 20, 20, 20, 20, 20,
+ 20, 20, 20, 20, 20, 20, 20, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
+ 21, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
+ 22, 22, 22, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
+ 23, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 25, 25, 25,
+ 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
+ 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 26, 26, 26, 26, 26, 26, 26,
+ 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
+ 26, 26, 26, 26, 26, 26, 26, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+ 27, 27, 28,
+};
+
+/* The order in which precode codeword lengths are stored */
+static const u8 deflate_precode_lens_permutation[DEFLATE_NUM_PRECODE_SYMS] = {
+ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15
+};
+
+/* Codewords for the DEFLATE Huffman codes. */
+struct deflate_codewords {
+ u32 litlen[DEFLATE_NUM_LITLEN_SYMS];
+ u32 offset[DEFLATE_NUM_OFFSET_SYMS];
+};
+
+/* Codeword lengths (in bits) for the DEFLATE Huffman codes.
+ * A zero length means the corresponding symbol had zero frequency. */
+struct deflate_lens {
+ u8 litlen[DEFLATE_NUM_LITLEN_SYMS];
+ u8 offset[DEFLATE_NUM_OFFSET_SYMS];
+};
+
+/* Codewords and lengths for the DEFLATE Huffman codes. */
+struct deflate_codes {
+ struct deflate_codewords codewords;
+ struct deflate_lens lens;
+};
+
+/* Symbol frequency counters for the DEFLATE Huffman codes. */
+struct deflate_freqs {
+ u32 litlen[DEFLATE_NUM_LITLEN_SYMS];
+ u32 offset[DEFLATE_NUM_OFFSET_SYMS];
+};
+
+#if SUPPORT_NEAR_OPTIMAL_PARSING
+
+/* Costs for the near-optimal parsing algorithm. */
+struct deflate_costs {
+
+ /* The cost to output each possible literal. */
+ u32 literal[DEFLATE_NUM_LITERALS];
+
+ /* The cost to output each possible match length. */
+ u32 length[DEFLATE_MAX_MATCH_LEN + 1];
+
+ /* The cost to output a match offset of each possible offset slot. */
+ u32 offset_slot[DEFLATE_NUM_OFFSET_SYMS];
+};
+
+/*
+ * COST_SHIFT is a scaling factor that makes it possible to consider fractional
+ * bit costs. A token requiring 'n' bits to represent has cost n << COST_SHIFT.
+ *
+ * Note: this is only useful as a statistical trick for when the true costs are
+ * unknown. In reality, each token in DEFLATE requires a whole number of bits
+ * to output.
+ */
+#define COST_SHIFT 3
+
+/*
+ * The NOSTAT_BITS value for a given alphabet is the number of bits assumed to
+ * be needed to output a symbol that was unused in the previous optimization
+ * pass. Assigning a default cost allows the symbol to be used in the next
+ * optimization pass. However, the cost should be relatively high because the
+ * symbol probably won't be used very many times (if at all).
+ */
+#define LITERAL_NOSTAT_BITS 13
+#define LENGTH_NOSTAT_BITS 13
+#define OFFSET_NOSTAT_BITS 10
+
+#endif /* SUPPORT_NEAR_OPTIMAL_PARSING */
+
+/*
+ * Represents a run of literals followed by a match or end-of-block. This
+ * struct is needed to temporarily store items chosen by the parser, since items
+ * cannot be written until all items for the block have been chosen and the
+ * block's Huffman codes have been computed.
+ */
+struct deflate_sequence {
+
+ /* Bits 0..22: the number of literals in this run. This may be 0 and
+ * can be at most about SOFT_MAX_BLOCK_LENGTH. The literals are not
+ * stored explicitly in this structure; instead, they are read directly
+ * from the uncompressed data.
+ *
+ * Bits 23..31: the length of the match which follows the literals, or 0
+ * if this literal run was the last in the block, so there is no match
+ * which follows it. */
+ u32 litrunlen_and_length;
+
+ /* If 'length' doesn't indicate end-of-block, then this is the offset of
+ * the match which follows the literals. */
+ u16 offset;
+
+ /* If 'length' doesn't indicate end-of-block, then this is the offset
+ * symbol of the match which follows the literals. */
+ u8 offset_symbol;
+
+ /* If 'length' doesn't indicate end-of-block, then this is the length
+ * slot of the match which follows the literals. */
+ u8 length_slot;
+};
+
+#if SUPPORT_NEAR_OPTIMAL_PARSING
+
+/*
+ * This structure represents a byte position in the input data and a node in the
+ * graph of possible match/literal choices for the current block.
+ *
+ * Logically, each incoming edge to this node is labeled with a literal or a
+ * match that can be taken to reach this position from an earlier position; and
+ * each outgoing edge from this node is labeled with a literal or a match that
+ * can be taken to advance from this position to a later position.
+ *
+ * But these "edges" are actually stored elsewhere (in 'match_cache'). Here we
+ * associate with each node just two pieces of information:
+ *
+ * 'cost_to_end' is the minimum cost to reach the end of the block from
+ * this position.
+ *
+ * 'item' represents the literal or match that must be chosen from here to
+ * reach the end of the block with the minimum cost. Equivalently, this
+ * can be interpreted as the label of the outgoing edge on the minimum-cost
+ * path to the "end of block" node from this node.
+ */
+struct deflate_optimum_node {
+
+ u32 cost_to_end;
+
+ /*
+ * Notes on the match/literal representation used here:
+ *
+ * The low bits of 'item' are the length: 1 if this is a literal,
+ * or the match length if this is a match.
+ *
+ * The high bits of 'item' are the actual literal byte if this is a
+ * literal, or the match offset if this is a match.
+ */
+#define OPTIMUM_OFFSET_SHIFT 9
+#define OPTIMUM_LEN_MASK (((u32)1 << OPTIMUM_OFFSET_SHIFT) - 1)
+ u32 item;
+
+};
+
+#endif /* SUPPORT_NEAR_OPTIMAL_PARSING */
+
+/* Block split statistics. See "Block splitting algorithm" below. */
+#define NUM_LITERAL_OBSERVATION_TYPES 8
+#define NUM_MATCH_OBSERVATION_TYPES 2
+#define NUM_OBSERVATION_TYPES (NUM_LITERAL_OBSERVATION_TYPES + NUM_MATCH_OBSERVATION_TYPES)
+struct block_split_stats {
+ u32 new_observations[NUM_OBSERVATION_TYPES];
+ u32 observations[NUM_OBSERVATION_TYPES];
+ u32 num_new_observations;
+ u32 num_observations;
+};
+
+/* The main DEFLATE compressor structure */
+struct libdeflate_compressor {
+
+ /* Pointer to the compress() implementation chosen at allocation time */
+ size_t (*impl)(struct libdeflate_compressor *,
+ const u8 *, size_t, u8 *, size_t);
+
+ /* Frequency counters for the current block */
+ struct deflate_freqs freqs;
+
+ /* Dynamic Huffman codes for the current block */
+ struct deflate_codes codes;
+
+ /* Static Huffman codes */
+ struct deflate_codes static_codes;
+
+ /* Block split statistics for the currently pending block */
+ struct block_split_stats split_stats;
+
+ /* A table for fast lookups of offset slot by match offset.
+ *
+ * If the full table is being used, it is a direct mapping from offset
+ * to offset slot.
+ *
+ * If the condensed table is being used, the first 256 entries map
+ * directly to the offset slots of offsets 1 through 256. The next 256
+ * entries map to the offset slots for the remaining offsets, stepping
+ * through the offsets with a stride of 128. This relies on the fact
+ * that each of the remaining offset slots contains at least 128 offsets
+ * and has an offset base that is a multiple of 128. */
+#if USE_FULL_OFFSET_SLOT_FAST
+ u8 offset_slot_fast[DEFLATE_MAX_MATCH_OFFSET + 1];
+#else
+ u8 offset_slot_fast[512];
+#endif
+
+ /* The "nice" match length: if a match of this length is found, choose
+ * it immediately without further consideration. */
+ unsigned nice_match_length;
+
+ /* The maximum search depth: consider at most this many potential
+ * matches at each position. */
+ unsigned max_search_depth;
+
+ /* The compression level with which this compressor was created. */
+ unsigned compression_level;
+
+ /* Anything smaller than this we won't bother trying to compress. */
+ unsigned min_size_to_compress;
+
+ /* Temporary space for Huffman code output */
+ u32 precode_freqs[DEFLATE_NUM_PRECODE_SYMS];
+ u8 precode_lens[DEFLATE_NUM_PRECODE_SYMS];
+ u32 precode_codewords[DEFLATE_NUM_PRECODE_SYMS];
+ unsigned precode_items[DEFLATE_NUM_LITLEN_SYMS + DEFLATE_NUM_OFFSET_SYMS];
+ unsigned num_litlen_syms;
+ unsigned num_offset_syms;
+ unsigned num_explicit_lens;
+ unsigned num_precode_items;
+
+ union {
+ /* Data for greedy or lazy parsing */
+ struct {
+ /* Hash chain matchfinder */
+ struct hc_matchfinder hc_mf;
+
+ /* The matches and literals that the parser has chosen
+ * for the current block. The required length of this
+ * array is limited by the maximum number of matches
+ * that can ever be chosen for a single block, plus one
+ * for the special entry at the end. */
+ struct deflate_sequence sequences[
+ DIV_ROUND_UP(SOFT_MAX_BLOCK_LENGTH,
+ DEFLATE_MIN_MATCH_LEN) + 1];
+ } g; /* (g)reedy */
+
+ #if SUPPORT_NEAR_OPTIMAL_PARSING
+ /* Data for near-optimal parsing */
+ struct {
+
+ /* Binary tree matchfinder */
+ struct bt_matchfinder bt_mf;
+
+ /*
+ * Cached matches for the current block. This array
+ * contains the matches that were found at each position
+ * in the block. Specifically, for each position, there
+ * is a list of matches found at that position, if any,
+ * sorted by strictly increasing length. In addition,
+ * following the matches for each position, there is a
+ * special 'struct lz_match' whose 'length' member
+ * contains the number of matches found at that
+ * position, and whose 'offset' member contains the
+ * literal at that position.
+ *
+ * Note: in rare cases, there will be a very high number
+ * of matches in the block and this array will overflow.
+ * If this happens, we force the end of the current
+ * block. CACHE_LENGTH is the length at which we
+ * actually check for overflow. The extra slots beyond
+ * this are enough to absorb the worst case overflow,
+ * which occurs if starting at &match_cache[CACHE_LENGTH
+ * - 1], we write MAX_MATCHES_PER_POS matches and a
+ * match count header, then skip searching for matches
+ * at 'DEFLATE_MAX_MATCH_LEN - 1' positions and write
+ * the match count header for each.
+ */
+ struct lz_match match_cache[CACHE_LENGTH +
+ MAX_MATCHES_PER_POS +
+ DEFLATE_MAX_MATCH_LEN - 1];
+
+ /*
+ * Array of nodes, one per position, for running the
+ * minimum-cost path algorithm.
+ *
+ * This array must be large enough to accommodate the
+ * worst-case number of nodes, which occurs if we find a
+ * match of length DEFLATE_MAX_MATCH_LEN at position
+ * SOFT_MAX_BLOCK_LENGTH - 1, producing a block of
+ * length SOFT_MAX_BLOCK_LENGTH - 1 +
+ * DEFLATE_MAX_MATCH_LEN. Add one for the end-of-block
+ * node.
+ */
+ struct deflate_optimum_node optimum_nodes[SOFT_MAX_BLOCK_LENGTH - 1 +
+ DEFLATE_MAX_MATCH_LEN + 1];
+
+ /* The current cost model being used. */
+ struct deflate_costs costs;
+
+ unsigned num_optim_passes;
+ } n; /* (n)ear-optimal */
+ #endif /* SUPPORT_NEAR_OPTIMAL_PARSING */
+
+ } p; /* (p)arser */
+};
+
+/*
+ * The type for the bitbuffer variable, which temporarily holds bits that are
+ * being packed into bytes and written to the output buffer. For best
+ * performance, this should have size equal to a machine word.
+ */
+typedef machine_word_t bitbuf_t;
+#define BITBUF_NBITS (8 * sizeof(bitbuf_t))
+
+/* Can the specified number of bits always be added to 'bitbuf' after any
+ * pending bytes have been flushed? */
+#define CAN_BUFFER(n) ((n) <= BITBUF_NBITS - 7)
+
+/*
+ * Structure to keep track of the current state of sending bits to the
+ * compressed output buffer.
+ */
+struct deflate_output_bitstream {
+
+ /* Bits that haven't yet been written to the output buffer. */
+ bitbuf_t bitbuf;
+
+ /* Number of bits currently held in @bitbuf. */
+ unsigned bitcount;
+
+ /* Pointer to the beginning of the output buffer. */
+ u8 *begin;
+
+ /* Pointer to the position in the output buffer at which the next byte
+ * should be written. */
+ u8 *next;
+
+ /* Pointer just past the end of the output buffer. */
+ u8 *end;
+};
+
+/*
+ * OUTPUT_END_PADDING is the size, in bytes, of the extra space that must be
+ * present following os->end, in order to not overrun the buffer when generating
+ * output. When UNALIGNED_ACCESS_IS_FAST, we need at least sizeof(bitbuf_t)
+ * bytes for put_unaligned_leword(). Otherwise we need only 1 byte. However,
+ * to make the compression algorithm produce the same result on all CPU
+ * architectures (which is sometimes desirable), we have to unconditionally use
+ * the maximum for any CPU, which is sizeof(bitbuf_t) == 8.
+ */
+#define OUTPUT_END_PADDING 8
+
+/* Initialize the output bitstream. 'size' is assumed to be at least
+ * OUTPUT_END_PADDING. */
+static void
+deflate_init_output(struct deflate_output_bitstream *os,
+ void *buffer, size_t size)
+{
+ os->bitbuf = 0;
+ os->bitcount = 0;
+ os->begin = buffer;
+ os->next = os->begin;
+ os->end = os->begin + size - OUTPUT_END_PADDING;
+}
+
+/* Add some bits to the bitbuffer variable of the output bitstream. The caller
+ * must make sure there is enough room. */
+static forceinline void
+deflate_add_bits(struct deflate_output_bitstream *os,
+ const bitbuf_t bits, const unsigned num_bits)
+{
+ os->bitbuf |= bits << os->bitcount;
+ os->bitcount += num_bits;
+}
+
+/* Flush bits from the bitbuffer variable to the output buffer. */
+static forceinline void
+deflate_flush_bits(struct deflate_output_bitstream *os)
+{
+ if (UNALIGNED_ACCESS_IS_FAST) {
+ /* Flush a whole word (branchlessly). */
+ put_unaligned_leword(os->bitbuf, os->next);
+ os->bitbuf >>= os->bitcount & ~7;
+ os->next += MIN(os->end - os->next, os->bitcount >> 3);
+ os->bitcount &= 7;
+ } else {
+ /* Flush a byte at a time. */
+ while (os->bitcount >= 8) {
+ *os->next = os->bitbuf;
+ if (os->next != os->end)
+ os->next++;
+ os->bitcount -= 8;
+ os->bitbuf >>= 8;
+ }
+ }
+}
+
+/* Align the bitstream on a byte boundary. */
+static forceinline void
+deflate_align_bitstream(struct deflate_output_bitstream *os)
+{
+ os->bitcount += -os->bitcount & 7;
+ deflate_flush_bits(os);
+}
+
+/*
+ * Flush any remaining bits to the output buffer if needed. Return the total
+ * number of bytes written to the output buffer, or 0 if an overflow occurred.
+ */
+static size_t
+deflate_flush_output(struct deflate_output_bitstream *os)
+{
+ if (os->next == os->end) /* overflow? */
+ return 0;
+
+ while ((int)os->bitcount > 0) {
+ *os->next++ = os->bitbuf;
+ os->bitcount -= 8;
+ os->bitbuf >>= 8;
+ }
+
+ return os->next - os->begin;
+}
+
+/* Given the binary tree node A[subtree_idx] whose children already
+ * satisfy the maxheap property, swap the node with its greater child
+ * until it is greater than both its children, so that the maxheap
+ * property is satisfied in the subtree rooted at A[subtree_idx]. */
+static void
+heapify_subtree(u32 A[], unsigned length, unsigned subtree_idx)
+{
+ unsigned parent_idx;
+ unsigned child_idx;
+ u32 v;
+
+ v = A[subtree_idx];
+ parent_idx = subtree_idx;
+ while ((child_idx = parent_idx * 2) <= length) {
+ if (child_idx < length && A[child_idx + 1] > A[child_idx])
+ child_idx++;
+ if (v >= A[child_idx])
+ break;
+ A[parent_idx] = A[child_idx];
+ parent_idx = child_idx;
+ }
+ A[parent_idx] = v;
+}
+
+/* Rearrange the array 'A' so that it satisfies the maxheap property.
+ * 'A' uses 1-based indices, so the children of A[i] are A[i*2] and A[i*2 + 1].
+ */
+static void
+heapify_array(u32 A[], unsigned length)
+{
+ unsigned subtree_idx;
+
+ for (subtree_idx = length / 2; subtree_idx >= 1; subtree_idx--)
+ heapify_subtree(A, length, subtree_idx);
+}
+
+/*
+ * Sort the array 'A', which contains 'length' unsigned 32-bit integers.
+ *
+ * Note: name this function heap_sort() instead of heapsort() to avoid colliding
+ * with heapsort() from stdlib.h on BSD-derived systems --- though this isn't
+ * necessary when compiling with -D_ANSI_SOURCE, which is the better solution.
+ */
+static void
+heap_sort(u32 A[], unsigned length)
+{
+ A--; /* Use 1-based indices */
+
+ heapify_array(A, length);
+
+ while (length >= 2) {
+ u32 tmp = A[length];
+ A[length] = A[1];
+ A[1] = tmp;
+ length--;
+ heapify_subtree(A, length, 1);
+ }
+}
+
+#define NUM_SYMBOL_BITS 10
+#define SYMBOL_MASK ((1 << NUM_SYMBOL_BITS) - 1)
+
+#define GET_NUM_COUNTERS(num_syms) ((((num_syms) + 3 / 4) + 3) & ~3)
+/*
+ * Sort the symbols primarily by frequency and secondarily by symbol
+ * value. Discard symbols with zero frequency and fill in an array with
+ * the remaining symbols, along with their frequencies. The low
+ * NUM_SYMBOL_BITS bits of each array entry will contain the symbol
+ * value, and the remaining bits will contain the frequency.
+ *
+ * @num_syms
+ * Number of symbols in the alphabet.
+ * Can't be greater than (1 << NUM_SYMBOL_BITS).
+ *
+ * @freqs[num_syms]
+ * The frequency of each symbol.
+ *
+ * @lens[num_syms]
+ * An array that eventually will hold the length of each codeword.
+ * This function only fills in the codeword lengths for symbols that
+ * have zero frequency, which are not well defined per se but will
+ * be set to 0.
+ *
+ * @symout[num_syms]
+ * The output array, described above.
+ *
+ * Returns the number of entries in 'symout' that were filled. This is
+ * the number of symbols that have nonzero frequency.
+ */
+static unsigned
+sort_symbols(unsigned num_syms, const u32 freqs[restrict],
+ u8 lens[restrict], u32 symout[restrict])
+{
+ unsigned sym;
+ unsigned i;
+ unsigned num_used_syms;
+ unsigned num_counters;
+ unsigned counters[GET_NUM_COUNTERS(DEFLATE_MAX_NUM_SYMS)];
+
+ /* We rely on heapsort, but with an added optimization. Since
+ * it's common for most symbol frequencies to be low, we first do
+ * a count sort using a limited number of counters. High
+ * frequencies will be counted in the last counter, and only they
+ * will be sorted with heapsort.
+ *
+ * Note: with more symbols, it is generally beneficial to have more
+ * counters. About 1 counter per 4 symbols seems fast.
+ *
+ * Note: I also tested radix sort, but even for large symbol
+ * counts (> 255) and frequencies bounded at 16 bits (enabling
+ * radix sort by just two base-256 digits), it didn't seem any
+ * faster than the method implemented here.
+ *
+ * Note: I tested the optimized quicksort implementation from
+ * glibc (with indirection overhead removed), but it was only
+ * marginally faster than the simple heapsort implemented here.
+ *
+ * Tests were done with building the codes for LZX. Results may
+ * vary for different compression algorithms...! */
+
+ num_counters = GET_NUM_COUNTERS(num_syms);
+
+ memset(counters, 0, num_counters * sizeof(counters[0]));
+
+ /* Count the frequencies. */
+ for (sym = 0; sym < num_syms; sym++)
+ counters[MIN(freqs[sym], num_counters - 1)]++;
+
+ /* Make the counters cumulative, ignoring the zero-th, which
+ * counted symbols with zero frequency. As a side effect, this
+ * calculates the number of symbols with nonzero frequency. */
+ num_used_syms = 0;
+ for (i = 1; i < num_counters; i++) {
+ unsigned count = counters[i];
+ counters[i] = num_used_syms;
+ num_used_syms += count;
+ }
+
+ /* Sort nonzero-frequency symbols using the counters. At the
+ * same time, set the codeword lengths of zero-frequency symbols
+ * to 0. */
+ for (sym = 0; sym < num_syms; sym++) {
+ u32 freq = freqs[sym];
+ if (freq != 0) {
+ symout[counters[MIN(freq, num_counters - 1)]++] =
+ sym | (freq << NUM_SYMBOL_BITS);
+ } else {
+ lens[sym] = 0;
+ }
+ }
+
+ /* Sort the symbols counted in the last counter. */
+ heap_sort(symout + counters[num_counters - 2],
+ counters[num_counters - 1] - counters[num_counters - 2]);
+
+ return num_used_syms;
+}
+
+/*
+ * Build the Huffman tree.
+ *
+ * This is an optimized implementation that
+ * (a) takes advantage of the frequencies being already sorted;
+ * (b) only generates non-leaf nodes, since the non-leaf nodes of a
+ * Huffman tree are sufficient to generate a canonical code;
+ * (c) Only stores parent pointers, not child pointers;
+ * (d) Produces the nodes in the same memory used for input
+ * frequency information.
+ *
+ * Array 'A', which contains 'sym_count' entries, is used for both input
+ * and output. For this function, 'sym_count' must be at least 2.
+ *
+ * For input, the array must contain the frequencies of the symbols,
+ * sorted in increasing order. Specifically, each entry must contain a
+ * frequency left shifted by NUM_SYMBOL_BITS bits. Any data in the low
+ * NUM_SYMBOL_BITS bits of the entries will be ignored by this function.
+ * Although these bits will, in fact, contain the symbols that correspond
+ * to the frequencies, this function is concerned with frequencies only
+ * and keeps the symbols as-is.
+ *
+ * For output, this function will produce the non-leaf nodes of the
+ * Huffman tree. These nodes will be stored in the first (sym_count - 1)
+ * entries of the array. Entry A[sym_count - 2] will represent the root
+ * node. Each other node will contain the zero-based index of its parent
+ * node in 'A', left shifted by NUM_SYMBOL_BITS bits. The low
+ * NUM_SYMBOL_BITS bits of each entry in A will be kept as-is. Again,
+ * note that although these low bits will, in fact, contain a symbol
+ * value, this symbol will have *no relationship* with the Huffman tree
+ * node that happens to occupy the same slot. This is because this
+ * implementation only generates the non-leaf nodes of the tree.
+ */
+static void
+build_tree(u32 A[], unsigned sym_count)
+{
+ /* Index, in 'A', of next lowest frequency symbol that has not
+ * yet been processed. */
+ unsigned i = 0;
+
+ /* Index, in 'A', of next lowest frequency parentless non-leaf
+ * node; or, if equal to 'e', then no such node exists yet. */
+ unsigned b = 0;
+
+ /* Index, in 'A', of next node to allocate as a non-leaf. */
+ unsigned e = 0;
+
+ do {
+ unsigned m, n;
+ u32 freq_shifted;
+
+ /* Choose the two next lowest frequency entries. */
+
+ if (i != sym_count &&
+ (b == e || (A[i] >> NUM_SYMBOL_BITS) <= (A[b] >> NUM_SYMBOL_BITS)))
+ m = i++;
+ else
+ m = b++;
+
+ if (i != sym_count &&
+ (b == e || (A[i] >> NUM_SYMBOL_BITS) <= (A[b] >> NUM_SYMBOL_BITS)))
+ n = i++;
+ else
+ n = b++;
+
+ /* Allocate a non-leaf node and link the entries to it.
+ *
+ * If we link an entry that we're visiting for the first
+ * time (via index 'i'), then we're actually linking a
+ * leaf node and it will have no effect, since the leaf
+ * will be overwritten with a non-leaf when index 'e'
+ * catches up to it. But it's not any slower to
+ * unconditionally set the parent index.
+ *
+ * We also compute the frequency of the non-leaf node as
+ * the sum of its two children's frequencies. */
+
+ freq_shifted = (A[m] & ~SYMBOL_MASK) + (A[n] & ~SYMBOL_MASK);
+
+ A[m] = (A[m] & SYMBOL_MASK) | (e << NUM_SYMBOL_BITS);
+ A[n] = (A[n] & SYMBOL_MASK) | (e << NUM_SYMBOL_BITS);
+ A[e] = (A[e] & SYMBOL_MASK) | freq_shifted;
+ e++;
+ } while (sym_count - e > 1);
+ /* When just one entry remains, it is a "leaf" that was
+ * linked to some other node. We ignore it, since the
+ * rest of the array contains the non-leaves which we
+ * need. (Note that we're assuming the cases with 0 or 1
+ * symbols were handled separately.) */
+}
+
+/*
+ * Given the stripped-down Huffman tree constructed by build_tree(),
+ * determine the number of codewords that should be assigned each
+ * possible length, taking into account the length-limited constraint.
+ *
+ * @A
+ * The array produced by build_tree(), containing parent index
+ * information for the non-leaf nodes of the Huffman tree. Each
+ * entry in this array is a node; a node's parent always has a
+ * greater index than that node itself. This function will
+ * overwrite the parent index information in this array, so
+ * essentially it will destroy the tree. However, the data in the
+ * low NUM_SYMBOL_BITS of each entry will be preserved.
+ *
+ * @root_idx
+ * The 0-based index of the root node in 'A', and consequently one
+ * less than the number of tree node entries in 'A'. (Or, really 2
+ * less than the actual length of 'A'.)
+ *
+ * @len_counts
+ * An array of length ('max_codeword_len' + 1) in which the number of
+ * codewords having each length <= max_codeword_len will be
+ * returned.
+ *
+ * @max_codeword_len
+ * The maximum permissible codeword length.
+ */
+static void
+compute_length_counts(u32 A[restrict], unsigned root_idx,
+ unsigned len_counts[restrict], unsigned max_codeword_len)
+{
+ unsigned len;
+ int node;
+
+ /* The key observations are:
+ *
+ * (1) We can traverse the non-leaf nodes of the tree, always
+ * visiting a parent before its children, by simply iterating
+ * through the array in reverse order. Consequently, we can
+ * compute the depth of each node in one pass, overwriting the
+ * parent indices with depths.
+ *
+ * (2) We can initially assume that in the real Huffman tree,
+ * both children of the root are leaves. This corresponds to two
+ * codewords of length 1. Then, whenever we visit a (non-leaf)
+ * node during the traversal, we modify this assumption to
+ * account for the current node *not* being a leaf, but rather
+ * its two children being leaves. This causes the loss of one
+ * codeword for the current depth and the addition of two
+ * codewords for the current depth plus one.
+ *
+ * (3) We can handle the length-limited constraint fairly easily
+ * by simply using the largest length available when a depth
+ * exceeds max_codeword_len.
+ */
+
+ for (len = 0; len <= max_codeword_len; len++)
+ len_counts[len] = 0;
+ len_counts[1] = 2;
+
+ /* Set the root node's depth to 0. */
+ A[root_idx] &= SYMBOL_MASK;
+
+ for (node = root_idx - 1; node >= 0; node--) {
+
+ /* Calculate the depth of this node. */
+
+ unsigned parent = A[node] >> NUM_SYMBOL_BITS;
+ unsigned parent_depth = A[parent] >> NUM_SYMBOL_BITS;
+ unsigned depth = parent_depth + 1;
+ unsigned len = depth;
+
+ /* Set the depth of this node so that it is available
+ * when its children (if any) are processed. */
+
+ A[node] = (A[node] & SYMBOL_MASK) | (depth << NUM_SYMBOL_BITS);
+
+ /* If needed, decrease the length to meet the
+ * length-limited constraint. This is not the optimal
+ * method for generating length-limited Huffman codes!
+ * But it should be good enough. */
+ if (len >= max_codeword_len) {
+ len = max_codeword_len;
+ do {
+ len--;
+ } while (len_counts[len] == 0);
+ }
+
+ /* Account for the fact that we have a non-leaf node at
+ * the current depth. */
+ len_counts[len]--;
+ len_counts[len + 1] += 2;
+ }
+}
+
+/*
+ * Generate the codewords for a canonical Huffman code.
+ *
+ * @A
+ * The output array for codewords. In addition, initially this
+ * array must contain the symbols, sorted primarily by frequency and
+ * secondarily by symbol value, in the low NUM_SYMBOL_BITS bits of
+ * each entry.
+ *
+ * @len
+ * Output array for codeword lengths.
+ *
+ * @len_counts
+ * An array that provides the number of codewords that will have
+ * each possible length <= max_codeword_len.
+ *
+ * @max_codeword_len
+ * Maximum length, in bits, of each codeword.
+ *
+ * @num_syms
+ * Number of symbols in the alphabet, including symbols with zero
+ * frequency. This is the length of the 'A' and 'len' arrays.
+ */
+static void
+gen_codewords(u32 A[restrict], u8 lens[restrict],
+ const unsigned len_counts[restrict],
+ unsigned max_codeword_len, unsigned num_syms)
+{
+ u32 next_codewords[DEFLATE_MAX_CODEWORD_LEN + 1];
+ unsigned i;
+ unsigned len;
+ unsigned sym;
+
+ /* Given the number of codewords that will have each length,
+ * assign codeword lengths to symbols. We do this by assigning
+ * the lengths in decreasing order to the symbols sorted
+ * primarily by increasing frequency and secondarily by
+ * increasing symbol value. */
+ for (i = 0, len = max_codeword_len; len >= 1; len--) {
+ unsigned count = len_counts[len];
+ while (count--)
+ lens[A[i++] & SYMBOL_MASK] = len;
+ }
+
+ /* Generate the codewords themselves. We initialize the
+ * 'next_codewords' array to provide the lexicographically first
+ * codeword of each length, then assign codewords in symbol
+ * order. This produces a canonical code. */
+ next_codewords[0] = 0;
+ next_codewords[1] = 0;
+ for (len = 2; len <= max_codeword_len; len++)
+ next_codewords[len] =
+ (next_codewords[len - 1] + len_counts[len - 1]) << 1;
+
+ for (sym = 0; sym < num_syms; sym++)
+ A[sym] = next_codewords[lens[sym]]++;
+}
+
+/*
+ * ---------------------------------------------------------------------
+ * make_canonical_huffman_code()
+ * ---------------------------------------------------------------------
+ *
+ * Given an alphabet and the frequency of each symbol in it, construct a
+ * length-limited canonical Huffman code.
+ *
+ * @num_syms
+ * The number of symbols in the alphabet. The symbols are the
+ * integers in the range [0, num_syms - 1]. This parameter must be
+ * at least 2 and can't be greater than (1 << NUM_SYMBOL_BITS).
+ *
+ * @max_codeword_len
+ * The maximum permissible codeword length.
+ *
+ * @freqs
+ * An array of @num_syms entries, each of which specifies the
+ * frequency of the corresponding symbol. It is valid for some,
+ * none, or all of the frequencies to be 0.
+ *
+ * @lens
+ * An array of @num_syms entries in which this function will return
+ * the length, in bits, of the codeword assigned to each symbol.
+ * Symbols with 0 frequency will not have codewords per se, but
+ * their entries in this array will be set to 0. No lengths greater
+ * than @max_codeword_len will be assigned.
+ *
+ * @codewords
+ * An array of @num_syms entries in which this function will return
+ * the codeword for each symbol, right-justified and padded on the
+ * left with zeroes. Codewords for symbols with 0 frequency will be
+ * undefined.
+ *
+ * ---------------------------------------------------------------------
+ *
+ * This function builds a length-limited canonical Huffman code.
+ *
+ * A length-limited Huffman code contains no codewords longer than some
+ * specified length, and has exactly (with some algorithms) or
+ * approximately (with the algorithm used here) the minimum weighted path
+ * length from the root, given this constraint.
+ *
+ * A canonical Huffman code satisfies the properties that a longer
+ * codeword never lexicographically precedes a shorter codeword, and the
+ * lexicographic ordering of codewords of the same length is the same as
+ * the lexicographic ordering of the corresponding symbols. A canonical
+ * Huffman code, or more generally a canonical prefix code, can be
+ * reconstructed from only a list containing the codeword length of each
+ * symbol.
+ *
+ * The classic algorithm to generate a Huffman code creates a node for
+ * each symbol, then inserts these nodes into a min-heap keyed by symbol
+ * frequency. Then, repeatedly, the two lowest-frequency nodes are
+ * removed from the min-heap and added as the children of a new node
+ * having frequency equal to the sum of its two children, which is then
+ * inserted into the min-heap. When only a single node remains in the
+ * min-heap, it is the root of the Huffman tree. The codeword for each
+ * symbol is determined by the path needed to reach the corresponding
+ * node from the root. Descending to the left child appends a 0 bit,
+ * whereas descending to the right child appends a 1 bit.
+ *
+ * The classic algorithm is relatively easy to understand, but it is
+ * subject to a number of inefficiencies. In practice, it is fastest to
+ * first sort the symbols by frequency. (This itself can be subject to
+ * an optimization based on the fact that most frequencies tend to be
+ * low.) At the same time, we sort secondarily by symbol value, which
+ * aids the process of generating a canonical code. Then, during tree
+ * construction, no heap is necessary because both the leaf nodes and the
+ * unparented non-leaf nodes can be easily maintained in sorted order.
+ * Consequently, there can never be more than two possibilities for the
+ * next-lowest-frequency node.
+ *
+ * In addition, because we're generating a canonical code, we actually
+ * don't need the leaf nodes of the tree at all, only the non-leaf nodes.
+ * This is because for canonical code generation we don't need to know
+ * where the symbols are in the tree. Rather, we only need to know how
+ * many leaf nodes have each depth (codeword length). And this
+ * information can, in fact, be quickly generated from the tree of
+ * non-leaves only.
+ *
+ * Furthermore, we can build this stripped-down Huffman tree directly in
+ * the array in which the codewords are to be generated, provided that
+ * these array slots are large enough to hold a symbol and frequency
+ * value.
+ *
+ * Still furthermore, we don't even need to maintain explicit child
+ * pointers. We only need the parent pointers, and even those can be
+ * overwritten in-place with depth information as part of the process of
+ * extracting codeword lengths from the tree. So in summary, we do NOT
+ * need a big structure like:
+ *
+ * struct huffman_tree_node {
+ * unsigned int symbol;
+ * unsigned int frequency;
+ * unsigned int depth;
+ * struct huffman_tree_node *left_child;
+ * struct huffman_tree_node *right_child;
+ * };
+ *
+ *
+ * ... which often gets used in "naive" implementations of Huffman code
+ * generation.
+ *
+ * Many of these optimizations are based on the implementation in 7-Zip
+ * (source file: C/HuffEnc.c), which has been placed in the public domain
+ * by Igor Pavlov.
+ */
+static void
+make_canonical_huffman_code(unsigned num_syms, unsigned max_codeword_len,
+ const u32 freqs[restrict],
+ u8 lens[restrict], u32 codewords[restrict])
+{
+ u32 *A = codewords;
+ unsigned num_used_syms;
+
+ STATIC_ASSERT(DEFLATE_MAX_NUM_SYMS <= 1 << NUM_SYMBOL_BITS);
+
+ /* We begin by sorting the symbols primarily by frequency and
+ * secondarily by symbol value. As an optimization, the array
+ * used for this purpose ('A') shares storage with the space in
+ * which we will eventually return the codewords. */
+
+ num_used_syms = sort_symbols(num_syms, freqs, lens, A);
+
+ /* 'num_used_syms' is the number of symbols with nonzero
+ * frequency. This may be less than @num_syms. 'num_used_syms'
+ * is also the number of entries in 'A' that are valid. Each
+ * entry consists of a distinct symbol and a nonzero frequency
+ * packed into a 32-bit integer. */
+
+ /* Handle special cases where only 0 or 1 symbols were used (had
+ * nonzero frequency). */
+
+ if (unlikely(num_used_syms == 0)) {
+ /* Code is empty. sort_symbols() already set all lengths
+ * to 0, so there is nothing more to do. */
+ return;
+ }
+
+ if (unlikely(num_used_syms == 1)) {
+ /* Only one symbol was used, so we only need one
+ * codeword. But two codewords are needed to form the
+ * smallest complete Huffman code, which uses codewords 0
+ * and 1. Therefore, we choose another symbol to which
+ * to assign a codeword. We use 0 (if the used symbol is
+ * not 0) or 1 (if the used symbol is 0). In either
+ * case, the lesser-valued symbol must be assigned
+ * codeword 0 so that the resulting code is canonical. */
+
+ unsigned sym = A[0] & SYMBOL_MASK;
+ unsigned nonzero_idx = sym ? sym : 1;
+
+ codewords[0] = 0;
+ lens[0] = 1;
+ codewords[nonzero_idx] = 1;
+ lens[nonzero_idx] = 1;
+ return;
+ }
+
+ /* Build a stripped-down version of the Huffman tree, sharing the
+ * array 'A' with the symbol values. Then extract length counts
+ * from the tree and use them to generate the final codewords. */
+
+ build_tree(A, num_used_syms);
+
+ {
+ unsigned len_counts[DEFLATE_MAX_CODEWORD_LEN + 1];
+
+ compute_length_counts(A, num_used_syms - 2,
+ len_counts, max_codeword_len);
+
+ gen_codewords(A, lens, len_counts, max_codeword_len, num_syms);
+ }
+}
+
+/*
+ * Clear the Huffman symbol frequency counters.
+ * This must be called when starting a new DEFLATE block.
+ */
+static void
+deflate_reset_symbol_frequencies(struct libdeflate_compressor *c)
+{
+ memset(&c->freqs, 0, sizeof(c->freqs));
+}
+
+/* Reverse the Huffman codeword 'codeword', which is 'len' bits in length. */
+static u32
+deflate_reverse_codeword(u32 codeword, u8 len)
+{
+ /* The following branchless algorithm is faster than going bit by bit.
+ * Note: since no codewords are longer than 16 bits, we only need to
+ * reverse the low 16 bits of the 'u32'. */
+ STATIC_ASSERT(DEFLATE_MAX_CODEWORD_LEN <= 16);
+
+ /* Flip adjacent 1-bit fields */
+ codeword = ((codeword & 0x5555) << 1) | ((codeword & 0xAAAA) >> 1);
+
+ /* Flip adjacent 2-bit fields */
+ codeword = ((codeword & 0x3333) << 2) | ((codeword & 0xCCCC) >> 2);
+
+ /* Flip adjacent 4-bit fields */
+ codeword = ((codeword & 0x0F0F) << 4) | ((codeword & 0xF0F0) >> 4);
+
+ /* Flip adjacent 8-bit fields */
+ codeword = ((codeword & 0x00FF) << 8) | ((codeword & 0xFF00) >> 8);
+
+ /* Return the high 'len' bits of the bit-reversed 16 bit value. */
+ return codeword >> (16 - len);
+}
+
+/* Make a canonical Huffman code with bit-reversed codewords. */
+static void
+deflate_make_huffman_code(unsigned num_syms, unsigned max_codeword_len,
+ const u32 freqs[], u8 lens[], u32 codewords[])
+{
+ unsigned sym;
+
+ make_canonical_huffman_code(num_syms, max_codeword_len,
+ freqs, lens, codewords);
+
+ for (sym = 0; sym < num_syms; sym++)
+ codewords[sym] = deflate_reverse_codeword(codewords[sym], lens[sym]);
+}
+
+/*
+ * Build the literal/length and offset Huffman codes for a DEFLATE block.
+ *
+ * This takes as input the frequency tables for each code and produces as output
+ * a set of tables that map symbols to codewords and codeword lengths.
+ */
+static void
+deflate_make_huffman_codes(const struct deflate_freqs *freqs,
+ struct deflate_codes *codes)
+{
+ STATIC_ASSERT(MAX_LITLEN_CODEWORD_LEN <= DEFLATE_MAX_LITLEN_CODEWORD_LEN);
+ STATIC_ASSERT(MAX_OFFSET_CODEWORD_LEN <= DEFLATE_MAX_OFFSET_CODEWORD_LEN);
+
+ deflate_make_huffman_code(DEFLATE_NUM_LITLEN_SYMS,
+ MAX_LITLEN_CODEWORD_LEN,
+ freqs->litlen,
+ codes->lens.litlen,
+ codes->codewords.litlen);
+
+ deflate_make_huffman_code(DEFLATE_NUM_OFFSET_SYMS,
+ MAX_OFFSET_CODEWORD_LEN,
+ freqs->offset,
+ codes->lens.offset,
+ codes->codewords.offset);
+}
+
+/* Initialize c->static_codes. */
+static void
+deflate_init_static_codes(struct libdeflate_compressor *c)
+{
+ unsigned i;
+
+ for (i = 0; i < 144; i++)
+ c->freqs.litlen[i] = 1 << (9 - 8);
+ for (; i < 256; i++)
+ c->freqs.litlen[i] = 1 << (9 - 9);
+ for (; i < 280; i++)
+ c->freqs.litlen[i] = 1 << (9 - 7);
+ for (; i < 288; i++)
+ c->freqs.litlen[i] = 1 << (9 - 8);
+
+ for (i = 0; i < 32; i++)
+ c->freqs.offset[i] = 1 << (5 - 5);
+
+ deflate_make_huffman_codes(&c->freqs, &c->static_codes);
+}
+
+/* Return the offset slot for the specified match offset. */
+static forceinline unsigned
+deflate_get_offset_slot(struct libdeflate_compressor *c, unsigned offset)
+{
+#if USE_FULL_OFFSET_SLOT_FAST
+ return c->offset_slot_fast[offset];
+#else
+ if (offset <= 256)
+ return c->offset_slot_fast[offset - 1];
+ else
+ return c->offset_slot_fast[256 + ((offset - 1) >> 7)];
+#endif
+}
+
+/* Write the header fields common to all DEFLATE block types. */
+static void
+deflate_write_block_header(struct deflate_output_bitstream *os,
+ bool is_final_block, unsigned block_type)
+{
+ deflate_add_bits(os, is_final_block, 1);
+ deflate_add_bits(os, block_type, 2);
+ deflate_flush_bits(os);
+}
+
+static unsigned
+deflate_compute_precode_items(const u8 lens[restrict],
+ const unsigned num_lens,
+ u32 precode_freqs[restrict],
+ unsigned precode_items[restrict])
+{
+ unsigned *itemptr;
+ unsigned run_start;
+ unsigned run_end;
+ unsigned extra_bits;
+ u8 len;
+
+ memset(precode_freqs, 0,
+ DEFLATE_NUM_PRECODE_SYMS * sizeof(precode_freqs[0]));
+
+ itemptr = precode_items;
+ run_start = 0;
+ do {
+ /* Find the next run of codeword lengths. */
+
+ /* len = the length being repeated */
+ len = lens[run_start];
+
+ /* Extend the run. */
+ run_end = run_start;
+ do {
+ run_end++;
+ } while (run_end != num_lens && len == lens[run_end]);
+
+ if (len == 0) {
+ /* Run of zeroes. */
+
+ /* Symbol 18: RLE 11 to 138 zeroes at a time. */
+ while ((run_end - run_start) >= 11) {
+ extra_bits = MIN((run_end - run_start) - 11, 0x7F);
+ precode_freqs[18]++;
+ *itemptr++ = 18 | (extra_bits << 5);
+ run_start += 11 + extra_bits;
+ }
+
+ /* Symbol 17: RLE 3 to 10 zeroes at a time. */
+ if ((run_end - run_start) >= 3) {
+ extra_bits = MIN((run_end - run_start) - 3, 0x7);
+ precode_freqs[17]++;
+ *itemptr++ = 17 | (extra_bits << 5);
+ run_start += 3 + extra_bits;
+ }
+ } else {
+
+ /* A run of nonzero lengths. */
+
+ /* Symbol 16: RLE 3 to 6 of the previous length. */
+ if ((run_end - run_start) >= 4) {
+ precode_freqs[len]++;
+ *itemptr++ = len;
+ run_start++;
+ do {
+ extra_bits = MIN((run_end - run_start) - 3, 0x3);
+ precode_freqs[16]++;
+ *itemptr++ = 16 | (extra_bits << 5);
+ run_start += 3 + extra_bits;
+ } while ((run_end - run_start) >= 3);
+ }
+ }
+
+ /* Output any remaining lengths without RLE. */
+ while (run_start != run_end) {
+ precode_freqs[len]++;
+ *itemptr++ = len;
+ run_start++;
+ }
+ } while (run_start != num_lens);
+
+ return itemptr - precode_items;
+}
+
+/*
+ * Huffman codeword lengths for dynamic Huffman blocks are compressed using a
+ * separate Huffman code, the "precode", which contains a symbol for each
+ * possible codeword length in the larger code as well as several special
+ * symbols to represent repeated codeword lengths (a form of run-length
+ * encoding). The precode is itself constructed in canonical form, and its
+ * codeword lengths are represented literally in 19 3-bit fields that
+ * immediately precede the compressed codeword lengths of the larger code.
+ */
+
+/* Precompute the information needed to output Huffman codes. */
+static void
+deflate_precompute_huffman_header(struct libdeflate_compressor *c)
+{
+ /* Compute how many litlen and offset symbols are needed. */
+
+ for (c->num_litlen_syms = DEFLATE_NUM_LITLEN_SYMS;
+ c->num_litlen_syms > 257;
+ c->num_litlen_syms--)
+ if (c->codes.lens.litlen[c->num_litlen_syms - 1] != 0)
+ break;
+
+ for (c->num_offset_syms = DEFLATE_NUM_OFFSET_SYMS;
+ c->num_offset_syms > 1;
+ c->num_offset_syms--)
+ if (c->codes.lens.offset[c->num_offset_syms - 1] != 0)
+ break;
+
+ /* If we're not using the full set of literal/length codeword lengths,
+ * then temporarily move the offset codeword lengths over so that the
+ * literal/length and offset codeword lengths are contiguous. */
+
+ STATIC_ASSERT(offsetof(struct deflate_lens, offset) ==
+ DEFLATE_NUM_LITLEN_SYMS);
+
+ if (c->num_litlen_syms != DEFLATE_NUM_LITLEN_SYMS) {
+ memmove((u8 *)&c->codes.lens + c->num_litlen_syms,
+ (u8 *)&c->codes.lens + DEFLATE_NUM_LITLEN_SYMS,
+ c->num_offset_syms);
+ }
+
+ /* Compute the "items" (RLE / literal tokens and extra bits) with which
+ * the codeword lengths in the larger code will be output. */
+ c->num_precode_items =
+ deflate_compute_precode_items((u8 *)&c->codes.lens,
+ c->num_litlen_syms +
+ c->num_offset_syms,
+ c->precode_freqs,
+ c->precode_items);
+
+ /* Build the precode. */
+ STATIC_ASSERT(MAX_PRE_CODEWORD_LEN <= DEFLATE_MAX_PRE_CODEWORD_LEN);
+ deflate_make_huffman_code(DEFLATE_NUM_PRECODE_SYMS,
+ MAX_PRE_CODEWORD_LEN,
+ c->precode_freqs, c->precode_lens,
+ c->precode_codewords);
+
+ /* Count how many precode lengths we actually need to output. */
+ for (c->num_explicit_lens = DEFLATE_NUM_PRECODE_SYMS;
+ c->num_explicit_lens > 4;
+ c->num_explicit_lens--)
+ if (c->precode_lens[deflate_precode_lens_permutation[
+ c->num_explicit_lens - 1]] != 0)
+ break;
+
+ /* Restore the offset codeword lengths if needed. */
+ if (c->num_litlen_syms != DEFLATE_NUM_LITLEN_SYMS) {
+ memmove((u8 *)&c->codes.lens + DEFLATE_NUM_LITLEN_SYMS,
+ (u8 *)&c->codes.lens + c->num_litlen_syms,
+ c->num_offset_syms);
+ }
+}
+
+/* Output the Huffman codes. */
+static void
+deflate_write_huffman_header(struct libdeflate_compressor *c,
+ struct deflate_output_bitstream *os)
+{
+ unsigned i;
+
+ deflate_add_bits(os, c->num_litlen_syms - 257, 5);
+ deflate_add_bits(os, c->num_offset_syms - 1, 5);
+ deflate_add_bits(os, c->num_explicit_lens - 4, 4);
+ deflate_flush_bits(os);
+
+ /* Output the lengths of the codewords in the precode. */
+ for (i = 0; i < c->num_explicit_lens; i++) {
+ deflate_add_bits(os, c->precode_lens[
+ deflate_precode_lens_permutation[i]], 3);
+ deflate_flush_bits(os);
+ }
+
+ /* Output the encoded lengths of the codewords in the larger code. */
+ for (i = 0; i < c->num_precode_items; i++) {
+ unsigned precode_item = c->precode_items[i];
+ unsigned precode_sym = precode_item & 0x1F;
+ deflate_add_bits(os, c->precode_codewords[precode_sym],
+ c->precode_lens[precode_sym]);
+ if (precode_sym >= 16) {
+ if (precode_sym == 16)
+ deflate_add_bits(os, precode_item >> 5, 2);
+ else if (precode_sym == 17)
+ deflate_add_bits(os, precode_item >> 5, 3);
+ else
+ deflate_add_bits(os, precode_item >> 5, 7);
+ }
+ STATIC_ASSERT(CAN_BUFFER(DEFLATE_MAX_PRE_CODEWORD_LEN + 7));
+ deflate_flush_bits(os);
+ }
+}
+
+static void
+deflate_write_sequences(struct deflate_output_bitstream * restrict os,
+ const struct deflate_codes * restrict codes,
+ const struct deflate_sequence sequences[restrict],
+ const u8 * restrict in_next)
+{
+ const struct deflate_sequence *seq = sequences;
+
+ for (;;) {
+ u32 litrunlen = seq->litrunlen_and_length & 0x7FFFFF;
+ unsigned length = seq->litrunlen_and_length >> 23;
+ unsigned length_slot;
+ unsigned litlen_symbol;
+ unsigned offset_symbol;
+
+ if (litrunlen) {
+ #if 1
+ while (litrunlen >= 4) {
+ unsigned lit0 = in_next[0];
+ unsigned lit1 = in_next[1];
+ unsigned lit2 = in_next[2];
+ unsigned lit3 = in_next[3];
+
+ deflate_add_bits(os, codes->codewords.litlen[lit0],
+ codes->lens.litlen[lit0]);
+ if (!CAN_BUFFER(2 * MAX_LITLEN_CODEWORD_LEN))
+ deflate_flush_bits(os);
+
+ deflate_add_bits(os, codes->codewords.litlen[lit1],
+ codes->lens.litlen[lit1]);
+ if (!CAN_BUFFER(4 * MAX_LITLEN_CODEWORD_LEN))
+ deflate_flush_bits(os);
+
+ deflate_add_bits(os, codes->codewords.litlen[lit2],
+ codes->lens.litlen[lit2]);
+ if (!CAN_BUFFER(2 * MAX_LITLEN_CODEWORD_LEN))
+ deflate_flush_bits(os);
+
+ deflate_add_bits(os, codes->codewords.litlen[lit3],
+ codes->lens.litlen[lit3]);
+ deflate_flush_bits(os);
+ in_next += 4;
+ litrunlen -= 4;
+ }
+ if (litrunlen-- != 0) {
+ deflate_add_bits(os, codes->codewords.litlen[*in_next],
+ codes->lens.litlen[*in_next]);
+ if (!CAN_BUFFER(3 * MAX_LITLEN_CODEWORD_LEN))
+ deflate_flush_bits(os);
+ in_next++;
+ if (litrunlen-- != 0) {
+ deflate_add_bits(os, codes->codewords.litlen[*in_next],
+ codes->lens.litlen[*in_next]);
+ if (!CAN_BUFFER(3 * MAX_LITLEN_CODEWORD_LEN))
+ deflate_flush_bits(os);
+ in_next++;
+ if (litrunlen-- != 0) {
+ deflate_add_bits(os, codes->codewords.litlen[*in_next],
+ codes->lens.litlen[*in_next]);
+ if (!CAN_BUFFER(3 * MAX_LITLEN_CODEWORD_LEN))
+ deflate_flush_bits(os);
+ in_next++;
+ }
+ }
+ if (CAN_BUFFER(3 * MAX_LITLEN_CODEWORD_LEN))
+ deflate_flush_bits(os);
+ }
+ #else
+ do {
+ unsigned lit = *in_next++;
+ deflate_add_bits(os, codes->codewords.litlen[lit],
+ codes->lens.litlen[lit]);
+ deflate_flush_bits(os);
+ } while (--litrunlen);
+ #endif
+ }
+
+ if (length == 0)
+ return;
+
+ in_next += length;
+
+ length_slot = seq->length_slot;
+ litlen_symbol = 257 + length_slot;
+
+ /* Litlen symbol */
+ deflate_add_bits(os, codes->codewords.litlen[litlen_symbol],
+ codes->lens.litlen[litlen_symbol]);
+
+ /* Extra length bits */
+ STATIC_ASSERT(CAN_BUFFER(MAX_LITLEN_CODEWORD_LEN +
+ DEFLATE_MAX_EXTRA_LENGTH_BITS));
+ deflate_add_bits(os, length - deflate_length_slot_base[length_slot],
+ deflate_extra_length_bits[length_slot]);
+
+ if (!CAN_BUFFER(MAX_LITLEN_CODEWORD_LEN +
+ DEFLATE_MAX_EXTRA_LENGTH_BITS +
+ MAX_OFFSET_CODEWORD_LEN +
+ DEFLATE_MAX_EXTRA_OFFSET_BITS))
+ deflate_flush_bits(os);
+
+ /* Offset symbol */
+ offset_symbol = seq->offset_symbol;
+ deflate_add_bits(os, codes->codewords.offset[offset_symbol],
+ codes->lens.offset[offset_symbol]);
+
+ if (!CAN_BUFFER(MAX_OFFSET_CODEWORD_LEN +
+ DEFLATE_MAX_EXTRA_OFFSET_BITS))
+ deflate_flush_bits(os);
+
+ /* Extra offset bits */
+ deflate_add_bits(os, seq->offset - deflate_offset_slot_base[offset_symbol],
+ deflate_extra_offset_bits[offset_symbol]);
+
+ deflate_flush_bits(os);
+
+ seq++;
+ }
+}
+
+#if SUPPORT_NEAR_OPTIMAL_PARSING
+/*
+ * Follow the minimum-cost path in the graph of possible match/literal choices
+ * for the current block and write out the matches/literals using the specified
+ * Huffman codes.
+ *
+ * Note: this is slightly duplicated with deflate_write_sequences(), the reason
+ * being that we don't want to waste time translating between intermediate
+ * match/literal representations.
+ */
+static void
+deflate_write_item_list(struct deflate_output_bitstream *os,
+ const struct deflate_codes *codes,
+ struct libdeflate_compressor *c,
+ u32 block_length)
+{
+ struct deflate_optimum_node *cur_node = &c->p.n.optimum_nodes[0];
+ struct deflate_optimum_node * const end_node = &c->p.n.optimum_nodes[block_length];
+ do {
+ unsigned length = cur_node->item & OPTIMUM_LEN_MASK;
+ unsigned offset = cur_node->item >> OPTIMUM_OFFSET_SHIFT;
+ unsigned litlen_symbol;
+ unsigned length_slot;
+ unsigned offset_slot;
+
+ if (length == 1) {
+ /* Literal */
+ litlen_symbol = offset;
+ deflate_add_bits(os, codes->codewords.litlen[litlen_symbol],
+ codes->lens.litlen[litlen_symbol]);
+ deflate_flush_bits(os);
+ } else {
+ /* Match length */
+ length_slot = deflate_length_slot[length];
+ litlen_symbol = 257 + length_slot;
+ deflate_add_bits(os, codes->codewords.litlen[litlen_symbol],
+ codes->lens.litlen[litlen_symbol]);
+
+ deflate_add_bits(os, length - deflate_length_slot_base[length_slot],
+ deflate_extra_length_bits[length_slot]);
+
+ if (!CAN_BUFFER(MAX_LITLEN_CODEWORD_LEN +
+ DEFLATE_MAX_EXTRA_LENGTH_BITS +
+ MAX_OFFSET_CODEWORD_LEN +
+ DEFLATE_MAX_EXTRA_OFFSET_BITS))
+ deflate_flush_bits(os);
+
+
+ /* Match offset */
+ offset_slot = deflate_get_offset_slot(c, offset);
+ deflate_add_bits(os, codes->codewords.offset[offset_slot],
+ codes->lens.offset[offset_slot]);
+
+ if (!CAN_BUFFER(MAX_OFFSET_CODEWORD_LEN +
+ DEFLATE_MAX_EXTRA_OFFSET_BITS))
+ deflate_flush_bits(os);
+
+ deflate_add_bits(os, offset - deflate_offset_slot_base[offset_slot],
+ deflate_extra_offset_bits[offset_slot]);
+
+ deflate_flush_bits(os);
+ }
+ cur_node += length;
+ } while (cur_node != end_node);
+}
+#endif /* SUPPORT_NEAR_OPTIMAL_PARSING */
+
+/* Output the end-of-block symbol. */
+static void
+deflate_write_end_of_block(struct deflate_output_bitstream *os,
+ const struct deflate_codes *codes)
+{
+ deflate_add_bits(os, codes->codewords.litlen[DEFLATE_END_OF_BLOCK],
+ codes->lens.litlen[DEFLATE_END_OF_BLOCK]);
+ deflate_flush_bits(os);
+}
+
+static void
+deflate_write_uncompressed_block(struct deflate_output_bitstream *os,
+ const u8 *data, u16 len,
+ bool is_final_block)
+{
+ deflate_write_block_header(os, is_final_block,
+ DEFLATE_BLOCKTYPE_UNCOMPRESSED);
+ deflate_align_bitstream(os);
+
+ if (4 + (u32)len >= os->end - os->next) {
+ os->next = os->end;
+ return;
+ }
+
+ put_unaligned_le16(len, os->next);
+ os->next += 2;
+ put_unaligned_le16(~len, os->next);
+ os->next += 2;
+ memcpy(os->next, data, len);
+ os->next += len;
+}
+
+static void
+deflate_write_uncompressed_blocks(struct deflate_output_bitstream *os,
+ const u8 *data, size_t data_length,
+ bool is_final_block)
+{
+ do {
+ u16 len = MIN(data_length, UINT16_MAX);
+
+ deflate_write_uncompressed_block(os, data, len,
+ is_final_block && len == data_length);
+ data += len;
+ data_length -= len;
+ } while (data_length != 0);
+}
+
+/*
+ * Choose the best type of block to use (dynamic Huffman, static Huffman, or
+ * uncompressed), then output it.
+ */
+static void
+deflate_flush_block(struct libdeflate_compressor * restrict c,
+ struct deflate_output_bitstream * restrict os,
+ const u8 * restrict block_begin, u32 block_length,
+ bool is_final_block, bool use_item_list)
+{
+ static const u8 deflate_extra_precode_bits[DEFLATE_NUM_PRECODE_SYMS] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 7,
+ };
+
+ /* Costs are measured in bits */
+ u32 dynamic_cost = 0;
+ u32 static_cost = 0;
+ u32 uncompressed_cost = 0;
+ struct deflate_codes *codes;
+ int block_type;
+ unsigned sym;
+
+ /* Tally the end-of-block symbol. */
+ c->freqs.litlen[DEFLATE_END_OF_BLOCK]++;
+
+ /* Build dynamic Huffman codes. */
+ deflate_make_huffman_codes(&c->freqs, &c->codes);
+
+ /* Account for the cost of sending dynamic Huffman codes. */
+ deflate_precompute_huffman_header(c);
+ dynamic_cost += 5 + 5 + 4 + (3 * c->num_explicit_lens);
+ for (sym = 0; sym < DEFLATE_NUM_PRECODE_SYMS; sym++) {
+ u32 extra = deflate_extra_precode_bits[sym];
+ dynamic_cost += c->precode_freqs[sym] *
+ (extra + c->precode_lens[sym]);
+ }
+
+ /* Account for the cost of encoding literals. */
+ for (sym = 0; sym < 256; sym++) {
+ dynamic_cost += c->freqs.litlen[sym] *
+ c->codes.lens.litlen[sym];
+ }
+ for (sym = 0; sym < 144; sym++)
+ static_cost += c->freqs.litlen[sym] * 8;
+ for (; sym < 256; sym++)
+ static_cost += c->freqs.litlen[sym] * 9;
+
+ /* Account for the cost of encoding the end-of-block symbol. */
+ dynamic_cost += c->codes.lens.litlen[256];
+ static_cost += 7;
+
+ /* Account for the cost of encoding lengths. */
+ for (sym = 257; sym < 257 + ARRAY_LEN(deflate_extra_length_bits); sym++) {
+ u32 extra = deflate_extra_length_bits[sym - 257];
+ dynamic_cost += c->freqs.litlen[sym] *
+ (extra + c->codes.lens.litlen[sym]);
+ static_cost += c->freqs.litlen[sym] *
+ (extra + c->static_codes.lens.litlen[sym]);
+ }
+
+ /* Account for the cost of encoding offsets. */
+ for (sym = 0; sym < ARRAY_LEN(deflate_extra_offset_bits); sym++) {
+ u32 extra = deflate_extra_offset_bits[sym];
+ dynamic_cost += c->freqs.offset[sym] *
+ (extra + c->codes.lens.offset[sym]);
+ static_cost += c->freqs.offset[sym] * (extra + 5);
+ }
+
+ /* Compute the cost of using uncompressed blocks. */
+ uncompressed_cost += (-(os->bitcount + 3) & 7) + 32 +
+ (40 * (DIV_ROUND_UP(block_length,
+ UINT16_MAX) - 1)) +
+ (8 * block_length);
+
+ /* Choose the cheapest block type. */
+ if (dynamic_cost < MIN(static_cost, uncompressed_cost)) {
+ block_type = DEFLATE_BLOCKTYPE_DYNAMIC_HUFFMAN;
+ codes = &c->codes;
+ } else if (static_cost < uncompressed_cost) {
+ block_type = DEFLATE_BLOCKTYPE_STATIC_HUFFMAN;
+ codes = &c->static_codes;
+ } else {
+ block_type = DEFLATE_BLOCKTYPE_UNCOMPRESSED;
+ }
+
+ /* Now actually output the block. */
+
+ if (block_type == DEFLATE_BLOCKTYPE_UNCOMPRESSED) {
+ /* Note: the length being flushed may exceed the maximum length
+ * of an uncompressed block (65535 bytes). Therefore, more than
+ * one uncompressed block might be needed. */
+ deflate_write_uncompressed_blocks(os, block_begin, block_length,
+ is_final_block);
+ } else {
+ /* Output the block header. */
+ deflate_write_block_header(os, is_final_block, block_type);
+
+ /* Output the Huffman codes (dynamic Huffman blocks only). */
+ if (block_type == DEFLATE_BLOCKTYPE_DYNAMIC_HUFFMAN)
+ deflate_write_huffman_header(c, os);
+
+ /* Output the literals, matches, and end-of-block symbol. */
+ #if SUPPORT_NEAR_OPTIMAL_PARSING
+ if (use_item_list)
+ deflate_write_item_list(os, codes, c, block_length);
+ else
+ #endif
+ deflate_write_sequences(os, codes, c->p.g.sequences,
+ block_begin);
+ deflate_write_end_of_block(os, codes);
+ }
+}
+
+static forceinline void
+deflate_choose_literal(struct libdeflate_compressor *c, unsigned literal,
+ u32 *litrunlen_p)
+{
+ c->freqs.litlen[literal]++;
+ ++*litrunlen_p;
+}
+
+static forceinline void
+deflate_choose_match(struct libdeflate_compressor *c,
+ unsigned length, unsigned offset,
+ u32 *litrunlen_p, struct deflate_sequence **next_seq_p)
+{
+ struct deflate_sequence *seq = *next_seq_p;
+ unsigned length_slot = deflate_length_slot[length];
+ unsigned offset_slot = deflate_get_offset_slot(c, offset);
+
+ c->freqs.litlen[257 + length_slot]++;
+ c->freqs.offset[offset_slot]++;
+
+ seq->litrunlen_and_length = ((u32)length << 23) | *litrunlen_p;
+ seq->offset = offset;
+ seq->length_slot = length_slot;
+ seq->offset_symbol = offset_slot;
+
+ *litrunlen_p = 0;
+ *next_seq_p = seq + 1;
+}
+
+static forceinline void
+deflate_finish_sequence(struct deflate_sequence *seq, u32 litrunlen)
+{
+ seq->litrunlen_and_length = litrunlen; /* length = 0 */
+}
+
+/******************************************************************************/
+
+/*
+ * Block splitting algorithm. The problem is to decide when it is worthwhile to
+ * start a new block with new Huffman codes. There is a theoretically optimal
+ * solution: recursively consider every possible block split, considering the
+ * exact cost of each block, and choose the minimum cost approach. But this is
+ * far too slow. Instead, as an approximation, we can count symbols and after
+ * every N symbols, compare the expected distribution of symbols based on the
+ * previous data with the actual distribution. If they differ "by enough", then
+ * start a new block.
+ *
+ * As an optimization and heuristic, we don't distinguish between every symbol
+ * but rather we combine many symbols into a single "observation type". For
+ * literals we only look at the high bits and low bits, and for matches we only
+ * look at whether the match is long or not. The assumption is that for typical
+ * "real" data, places that are good block boundaries will tend to be noticeable
+ * based only on changes in these aggregate frequencies, without looking for
+ * subtle differences in individual symbols. For example, a change from ASCII
+ * bytes to non-ASCII bytes, or from few matches (generally less compressible)
+ * to many matches (generally more compressible), would be easily noticed based
+ * on the aggregates.
+ *
+ * For determining whether the frequency distributions are "different enough" to
+ * start a new block, the simply heuristic of splitting when the sum of absolute
+ * differences exceeds a constant seems to be good enough. We also add a number
+ * proportional to the block length so that the algorithm is more likely to end
+ * long blocks than short blocks. This reflects the general expectation that it
+ * will become increasingly beneficial to start a new block as the current
+ * block grows longer.
+ *
+ * Finally, for an approximation, it is not strictly necessary that the exact
+ * symbols being used are considered. With "near-optimal parsing", for example,
+ * the actual symbols that will be used are unknown until after the block
+ * boundary is chosen and the block has been optimized. Since the final choices
+ * cannot be used, we can use preliminary "greedy" choices instead.
+ */
+
+/* Initialize the block split statistics when starting a new block. */
+static void
+init_block_split_stats(struct block_split_stats *stats)
+{
+ int i;
+
+ for (i = 0; i < NUM_OBSERVATION_TYPES; i++) {
+ stats->new_observations[i] = 0;
+ stats->observations[i] = 0;
+ }
+ stats->num_new_observations = 0;
+ stats->num_observations = 0;
+}
+
+/* Literal observation. Heuristic: use the top 2 bits and low 1 bits of the
+ * literal, for 8 possible literal observation types. */
+static forceinline void
+observe_literal(struct block_split_stats *stats, u8 lit)
+{
+ stats->new_observations[((lit >> 5) & 0x6) | (lit & 1)]++;
+ stats->num_new_observations++;
+}
+
+/* Match observation. Heuristic: use one observation type for "short match" and
+ * one observation type for "long match". */
+static forceinline void
+observe_match(struct block_split_stats *stats, unsigned length)
+{
+ stats->new_observations[NUM_LITERAL_OBSERVATION_TYPES + (length >= 9)]++;
+ stats->num_new_observations++;
+}
+
+static bool
+do_end_block_check(struct block_split_stats *stats, u32 block_length)
+{
+ int i;
+
+ if (stats->num_observations > 0) {
+
+ /* Note: to avoid slow divisions, we do not divide by
+ * 'num_observations', but rather do all math with the numbers
+ * multiplied by 'num_observations'. */
+ u32 total_delta = 0;
+ for (i = 0; i < NUM_OBSERVATION_TYPES; i++) {
+ u32 expected = stats->observations[i] * stats->num_new_observations;
+ u32 actual = stats->new_observations[i] * stats->num_observations;
+ u32 delta = (actual > expected) ? actual - expected :
+ expected - actual;
+ total_delta += delta;
+ }
+
+ /* Ready to end the block? */
+ if (total_delta + (block_length / 4096) * stats->num_observations >=
+ NUM_OBSERVATIONS_PER_BLOCK_CHECK * 200 / 512 * stats->num_observations)
+ return true;
+ }
+
+ for (i = 0; i < NUM_OBSERVATION_TYPES; i++) {
+ stats->num_observations += stats->new_observations[i];
+ stats->observations[i] += stats->new_observations[i];
+ stats->new_observations[i] = 0;
+ }
+ stats->num_new_observations = 0;
+ return false;
+}
+
+static forceinline bool
+should_end_block(struct block_split_stats *stats,
+ const u8 *in_block_begin, const u8 *in_next, const u8 *in_end)
+{
+ /* Ready to check block split statistics? */
+ if (stats->num_new_observations < NUM_OBSERVATIONS_PER_BLOCK_CHECK ||
+ in_next - in_block_begin < MIN_BLOCK_LENGTH ||
+ in_end - in_next < MIN_BLOCK_LENGTH)
+ return false;
+
+ return do_end_block_check(stats, in_next - in_block_begin);
+}
+
+/******************************************************************************/
+
+/*
+ * This is the level 0 "compressor". It always outputs uncompressed blocks.
+ */
+static size_t
+deflate_compress_none(struct libdeflate_compressor * restrict c,
+ const u8 * restrict in, size_t in_nbytes,
+ u8 * restrict out, size_t out_nbytes_avail)
+{
+ struct deflate_output_bitstream os;
+
+ deflate_init_output(&os, out, out_nbytes_avail);
+
+ deflate_write_uncompressed_blocks(&os, in, in_nbytes, true);
+
+ return deflate_flush_output(&os);
+}
+
+/*
+ * This is the "greedy" DEFLATE compressor. It always chooses the longest match.
+ */
+static size_t
+deflate_compress_greedy(struct libdeflate_compressor * restrict c,
+ const u8 * restrict in, size_t in_nbytes,
+ u8 * restrict out, size_t out_nbytes_avail)
+{
+ const u8 *in_next = in;
+ const u8 *in_end = in_next + in_nbytes;
+ struct deflate_output_bitstream os;
+ const u8 *in_cur_base = in_next;
+ unsigned max_len = DEFLATE_MAX_MATCH_LEN;
+ unsigned nice_len = MIN(c->nice_match_length, max_len);
+ u32 next_hashes[2] = {0, 0};
+
+ deflate_init_output(&os, out, out_nbytes_avail);
+ hc_matchfinder_init(&c->p.g.hc_mf);
+
+ do {
+ /* Starting a new DEFLATE block. */
+
+ const u8 * const in_block_begin = in_next;
+ const u8 * const in_max_block_end =
+ in_next + MIN(in_end - in_next, SOFT_MAX_BLOCK_LENGTH);
+ u32 litrunlen = 0;
+ struct deflate_sequence *next_seq = c->p.g.sequences;
+
+ init_block_split_stats(&c->split_stats);
+ deflate_reset_symbol_frequencies(c);
+
+ do {
+ u32 length;
+ u32 offset;
+
+ /* Decrease the maximum and nice match lengths if we're
+ * approaching the end of the input buffer. */
+ if (unlikely(max_len > in_end - in_next)) {
+ max_len = in_end - in_next;
+ nice_len = MIN(nice_len, max_len);
+ }
+
+ length = hc_matchfinder_longest_match(&c->p.g.hc_mf,
+ &in_cur_base,
+ in_next,
+ DEFLATE_MIN_MATCH_LEN - 1,
+ max_len,
+ nice_len,
+ c->max_search_depth,
+ next_hashes,
+ &offset);
+
+ if (length >= DEFLATE_MIN_MATCH_LEN) {
+ /* Match found. */
+ deflate_choose_match(c, length, offset,
+ &litrunlen, &next_seq);
+ observe_match(&c->split_stats, length);
+ in_next = hc_matchfinder_skip_positions(&c->p.g.hc_mf,
+ &in_cur_base,
+ in_next + 1,
+ in_end,
+ length - 1,
+ next_hashes);
+ } else {
+ /* No match found. */
+ deflate_choose_literal(c, *in_next, &litrunlen);
+ observe_literal(&c->split_stats, *in_next);
+ in_next++;
+ }
+
+ /* Check if it's time to output another block. */
+ } while (in_next < in_max_block_end &&
+ !should_end_block(&c->split_stats, in_block_begin, in_next, in_end));
+
+ deflate_finish_sequence(next_seq, litrunlen);
+ deflate_flush_block(c, &os, in_block_begin,
+ in_next - in_block_begin,
+ in_next == in_end, false);
+ } while (in_next != in_end);
+
+ return deflate_flush_output(&os);
+}
+
+/*
+ * This is the "lazy" DEFLATE compressor. Before choosing a match, it checks to
+ * see if there's a longer match at the next position. If yes, it outputs a
+ * literal and continues to the next position. If no, it outputs the match.
+ */
+static size_t
+deflate_compress_lazy(struct libdeflate_compressor * restrict c,
+ const u8 * restrict in, size_t in_nbytes,
+ u8 * restrict out, size_t out_nbytes_avail)
+{
+ const u8 *in_next = in;
+ const u8 *in_end = in_next + in_nbytes;
+ struct deflate_output_bitstream os;
+ const u8 *in_cur_base = in_next;
+ unsigned max_len = DEFLATE_MAX_MATCH_LEN;
+ unsigned nice_len = MIN(c->nice_match_length, max_len);
+ u32 next_hashes[2] = {0, 0};
+
+ deflate_init_output(&os, out, out_nbytes_avail);
+ hc_matchfinder_init(&c->p.g.hc_mf);
+
+ do {
+ /* Starting a new DEFLATE block. */
+
+ const u8 * const in_block_begin = in_next;
+ const u8 * const in_max_block_end =
+ in_next + MIN(in_end - in_next, SOFT_MAX_BLOCK_LENGTH);
+ u32 litrunlen = 0;
+ struct deflate_sequence *next_seq = c->p.g.sequences;
+
+ init_block_split_stats(&c->split_stats);
+ deflate_reset_symbol_frequencies(c);
+
+ do {
+ unsigned cur_len;
+ unsigned cur_offset;
+ unsigned next_len;
+ unsigned next_offset;
+
+ if (unlikely(in_end - in_next < DEFLATE_MAX_MATCH_LEN)) {
+ max_len = in_end - in_next;
+ nice_len = MIN(nice_len, max_len);
+ }
+
+ /* Find the longest match at the current position. */
+ cur_len = hc_matchfinder_longest_match(&c->p.g.hc_mf,
+ &in_cur_base,
+ in_next,
+ DEFLATE_MIN_MATCH_LEN - 1,
+ max_len,
+ nice_len,
+ c->max_search_depth,
+ next_hashes,
+ &cur_offset);
+ in_next += 1;
+
+ if (cur_len < DEFLATE_MIN_MATCH_LEN) {
+ /* No match found. Choose a literal. */
+ deflate_choose_literal(c, *(in_next - 1), &litrunlen);
+ observe_literal(&c->split_stats, *(in_next - 1));
+ continue;
+ }
+
+ have_cur_match:
+ observe_match(&c->split_stats, cur_len);
+
+ /* We have a match at the current position. */
+
+ /* If the current match is very long, choose it
+ * immediately. */
+ if (cur_len >= nice_len) {
+ deflate_choose_match(c, cur_len, cur_offset,
+ &litrunlen, &next_seq);
+ in_next = hc_matchfinder_skip_positions(&c->p.g.hc_mf,
+ &in_cur_base,
+ in_next,
+ in_end,
+ cur_len - 1,
+ next_hashes);
+ continue;
+ }
+
+ /*
+ * Try to find a match at the next position.
+ *
+ * Note: since we already have a match at the *current*
+ * position, we use only half the 'max_search_depth'
+ * when checking the *next* position. This is a useful
+ * trade-off because it's more worthwhile to use a
+ * greater search depth on the initial match.
+ *
+ * Note: it's possible to structure the code such that
+ * there's only one call to longest_match(), which
+ * handles both the "find the initial match" and "try to
+ * find a longer match" cases. However, it is faster to
+ * have two call sites, with longest_match() inlined at
+ * each.
+ */
+ if (unlikely(in_end - in_next < DEFLATE_MAX_MATCH_LEN)) {
+ max_len = in_end - in_next;
+ nice_len = MIN(nice_len, max_len);
+ }
+ next_len = hc_matchfinder_longest_match(&c->p.g.hc_mf,
+ &in_cur_base,
+ in_next,
+ cur_len,
+ max_len,
+ nice_len,
+ c->max_search_depth / 2,
+ next_hashes,
+ &next_offset);
+ in_next += 1;
+
+ if (next_len > cur_len) {
+ /* Found a longer match at the next position.
+ * Output a literal. Then the next match
+ * becomes the current match. */
+ deflate_choose_literal(c, *(in_next - 2), &litrunlen);
+ cur_len = next_len;
+ cur_offset = next_offset;
+ goto have_cur_match;
+ }
+
+ /* No longer match at the next position.
+ * Output the current match. */
+ deflate_choose_match(c, cur_len, cur_offset,
+ &litrunlen, &next_seq);
+ in_next = hc_matchfinder_skip_positions(&c->p.g.hc_mf,
+ &in_cur_base,
+ in_next,
+ in_end,
+ cur_len - 2,
+ next_hashes);
+
+ /* Check if it's time to output another block. */
+ } while (in_next < in_max_block_end &&
+ !should_end_block(&c->split_stats, in_block_begin, in_next, in_end));
+
+ deflate_finish_sequence(next_seq, litrunlen);
+ deflate_flush_block(c, &os, in_block_begin,
+ in_next - in_block_begin,
+ in_next == in_end, false);
+ } while (in_next != in_end);
+
+ return deflate_flush_output(&os);
+}
+
+#if SUPPORT_NEAR_OPTIMAL_PARSING
+
+/*
+ * Follow the minimum-cost path in the graph of possible match/literal choices
+ * for the current block and compute the frequencies of the Huffman symbols that
+ * would be needed to output those matches and literals.
+ */
+static void
+deflate_tally_item_list(struct libdeflate_compressor *c, u32 block_length)
+{
+ struct deflate_optimum_node *cur_node = &c->p.n.optimum_nodes[0];
+ struct deflate_optimum_node *end_node = &c->p.n.optimum_nodes[block_length];
+ do {
+ unsigned length = cur_node->item & OPTIMUM_LEN_MASK;
+ unsigned offset = cur_node->item >> OPTIMUM_OFFSET_SHIFT;
+
+ if (length == 1) {
+ /* Literal */
+ c->freqs.litlen[offset]++;
+ } else {
+ /* Match */
+ c->freqs.litlen[257 + deflate_length_slot[length]]++;
+ c->freqs.offset[deflate_get_offset_slot(c, offset)]++;
+ }
+ cur_node += length;
+ } while (cur_node != end_node);
+}
+
+/* Set the current cost model from the codeword lengths specified in @lens. */
+static void
+deflate_set_costs_from_codes(struct libdeflate_compressor *c,
+ const struct deflate_lens *lens)
+{
+ unsigned i;
+
+ /* Literals */
+ for (i = 0; i < DEFLATE_NUM_LITERALS; i++) {
+ u32 bits = (lens->litlen[i] ? lens->litlen[i] : LITERAL_NOSTAT_BITS);
+ c->p.n.costs.literal[i] = bits << COST_SHIFT;
+ }
+
+ /* Lengths */
+ for (i = DEFLATE_MIN_MATCH_LEN; i <= DEFLATE_MAX_MATCH_LEN; i++) {
+ unsigned length_slot = deflate_length_slot[i];
+ unsigned litlen_sym = 257 + length_slot;
+ u32 bits = (lens->litlen[litlen_sym] ? lens->litlen[litlen_sym] : LENGTH_NOSTAT_BITS);
+ bits += deflate_extra_length_bits[length_slot];
+ c->p.n.costs.length[i] = bits << COST_SHIFT;
+ }
+
+ /* Offset slots */
+ for (i = 0; i < ARRAY_LEN(deflate_offset_slot_base); i++) {
+ u32 bits = (lens->offset[i] ? lens->offset[i] : OFFSET_NOSTAT_BITS);
+ bits += deflate_extra_offset_bits[i];
+ c->p.n.costs.offset_slot[i] = bits << COST_SHIFT;
+ }
+}
+
+static forceinline u32
+deflate_default_literal_cost(unsigned literal)
+{
+ STATIC_ASSERT(COST_SHIFT == 3);
+ /* 66 is 8.25 bits/symbol */
+ return 66;
+}
+
+static forceinline u32
+deflate_default_length_slot_cost(unsigned length_slot)
+{
+ STATIC_ASSERT(COST_SHIFT == 3);
+ /* 60 is 7.5 bits/symbol */
+ return 60 + ((u32)deflate_extra_length_bits[length_slot] << COST_SHIFT);
+}
+
+static forceinline u32
+deflate_default_offset_slot_cost(unsigned offset_slot)
+{
+ STATIC_ASSERT(COST_SHIFT == 3);
+ /* 39 is 4.875 bits/symbol */
+ return 39 + ((u32)deflate_extra_offset_bits[offset_slot] << COST_SHIFT);
+}
+
+/*
+ * Set default symbol costs for the first block's first optimization pass.
+ *
+ * It works well to assume that each symbol is equally probable. This results
+ * in each symbol being assigned a cost of (-log2(1.0/num_syms) * (1 <<
+ * COST_SHIFT)) where 'num_syms' is the number of symbols in the corresponding
+ * alphabet. However, we intentionally bias the parse towards matches rather
+ * than literals by using a slightly lower default cost for length symbols than
+ * for literals. This often improves the compression ratio slightly.
+ */
+static void
+deflate_set_default_costs(struct libdeflate_compressor *c)
+{
+ unsigned i;
+
+ /* Literals */
+ for (i = 0; i < DEFLATE_NUM_LITERALS; i++)
+ c->p.n.costs.literal[i] = deflate_default_literal_cost(i);
+
+ /* Lengths */
+ for (i = DEFLATE_MIN_MATCH_LEN; i <= DEFLATE_MAX_MATCH_LEN; i++)
+ c->p.n.costs.length[i] = deflate_default_length_slot_cost(
+ deflate_length_slot[i]);
+
+ /* Offset slots */
+ for (i = 0; i < ARRAY_LEN(deflate_offset_slot_base); i++)
+ c->p.n.costs.offset_slot[i] = deflate_default_offset_slot_cost(i);
+}
+
+static forceinline void
+deflate_adjust_cost(u32 *cost_p, u32 default_cost)
+{
+ *cost_p += ((s32)default_cost - (s32)*cost_p) >> 1;
+}
+
+/*
+ * Adjust the costs when beginning a new block.
+ *
+ * Since the current costs have been optimized for the data, it's undesirable to
+ * throw them away and start over with the default costs. At the same time, we
+ * don't want to bias the parse by assuming that the next block will be similar
+ * to the current block. As a compromise, make the costs closer to the
+ * defaults, but don't simply set them to the defaults.
+ */
+static void
+deflate_adjust_costs(struct libdeflate_compressor *c)
+{
+ unsigned i;
+
+ /* Literals */
+ for (i = 0; i < DEFLATE_NUM_LITERALS; i++)
+ deflate_adjust_cost(&c->p.n.costs.literal[i],
+ deflate_default_literal_cost(i));
+
+ /* Lengths */
+ for (i = DEFLATE_MIN_MATCH_LEN; i <= DEFLATE_MAX_MATCH_LEN; i++)
+ deflate_adjust_cost(&c->p.n.costs.length[i],
+ deflate_default_length_slot_cost(
+ deflate_length_slot[i]));
+
+ /* Offset slots */
+ for (i = 0; i < ARRAY_LEN(deflate_offset_slot_base); i++)
+ deflate_adjust_cost(&c->p.n.costs.offset_slot[i],
+ deflate_default_offset_slot_cost(i));
+}
+
+/*
+ * Find the minimum-cost path through the graph of possible match/literal
+ * choices for this block.
+ *
+ * We find the minimum cost path from 'c->p.n.optimum_nodes[0]', which
+ * represents the node at the beginning of the block, to
+ * 'c->p.n.optimum_nodes[block_length]', which represents the node at the end of
+ * the block. Edge costs are evaluated using the cost model 'c->p.n.costs'.
+ *
+ * The algorithm works backwards, starting at the end node and proceeding
+ * backwards one node at a time. At each node, the minimum cost to reach the
+ * end node is computed and the match/literal choice that begins that path is
+ * saved.
+ */
+static void
+deflate_find_min_cost_path(struct libdeflate_compressor *c,
+ const u32 block_length,
+ const struct lz_match *cache_ptr)
+{
+ struct deflate_optimum_node *end_node = &c->p.n.optimum_nodes[block_length];
+ struct deflate_optimum_node *cur_node = end_node;
+
+ cur_node->cost_to_end = 0;
+ do {
+ unsigned num_matches;
+ unsigned literal;
+ u32 best_cost_to_end;
+
+ cur_node--;
+ cache_ptr--;
+
+ num_matches = cache_ptr->length;
+ literal = cache_ptr->offset;
+
+ /* It's always possible to choose a literal. */
+ best_cost_to_end = c->p.n.costs.literal[literal] +
+ (cur_node + 1)->cost_to_end;
+ cur_node->item = ((u32)literal << OPTIMUM_OFFSET_SHIFT) | 1;
+
+ /* Also consider matches if there are any. */
+ if (num_matches) {
+ const struct lz_match *match;
+ unsigned len;
+ unsigned offset;
+ unsigned offset_slot;
+ u32 offset_cost;
+ u32 cost_to_end;
+
+ /*
+ * Consider each length from the minimum
+ * (DEFLATE_MIN_MATCH_LEN) to the length of the longest
+ * match found at this position. For each length, we
+ * consider only the smallest offset for which that
+ * length is available. Although this is not guaranteed
+ * to be optimal due to the possibility of a larger
+ * offset costing less than a smaller offset to code,
+ * this is a very useful heuristic.
+ */
+ match = cache_ptr - num_matches;
+ len = DEFLATE_MIN_MATCH_LEN;
+ do {
+ offset = match->offset;
+ offset_slot = deflate_get_offset_slot(c, offset);
+ offset_cost = c->p.n.costs.offset_slot[offset_slot];
+ do {
+ cost_to_end = offset_cost +
+ c->p.n.costs.length[len] +
+ (cur_node + len)->cost_to_end;
+ if (cost_to_end < best_cost_to_end) {
+ best_cost_to_end = cost_to_end;
+ cur_node->item = ((u32)offset << OPTIMUM_OFFSET_SHIFT) | len;
+ }
+ } while (++len <= match->length);
+ } while (++match != cache_ptr);
+ cache_ptr -= num_matches;
+ }
+ cur_node->cost_to_end = best_cost_to_end;
+ } while (cur_node != &c->p.n.optimum_nodes[0]);
+}
+
+/*
+ * Choose the literal/match sequence to use for the current block. The basic
+ * algorithm finds a minimum-cost path through the block's graph of
+ * literal/match choices, given a cost model. However, the cost of each symbol
+ * is unknown until the Huffman codes have been built, but at the same time the
+ * Huffman codes depend on the frequencies of chosen symbols. Consequently,
+ * multiple passes must be used to try to approximate an optimal solution. The
+ * first pass uses default costs, mixed with the costs from the previous block
+ * if any. Later passes use the Huffman codeword lengths from the previous pass
+ * as the costs.
+ */
+static void
+deflate_optimize_block(struct libdeflate_compressor *c, u32 block_length,
+ const struct lz_match *cache_ptr, bool is_first_block)
+{
+ unsigned num_passes_remaining = c->p.n.num_optim_passes;
+ u32 i;
+
+ /* Force the block to really end at the desired length, even if some
+ * matches extend beyond it. */
+ for (i = block_length; i <= MIN(block_length - 1 + DEFLATE_MAX_MATCH_LEN,
+ ARRAY_LEN(c->p.n.optimum_nodes) - 1); i++)
+ c->p.n.optimum_nodes[i].cost_to_end = 0x80000000;
+
+ /* Set the initial costs. */
+ if (is_first_block)
+ deflate_set_default_costs(c);
+ else
+ deflate_adjust_costs(c);
+
+ for (;;) {
+ /* Find the minimum cost path for this pass. */
+ deflate_find_min_cost_path(c, block_length, cache_ptr);
+
+ /* Compute frequencies of the chosen symbols. */
+ deflate_reset_symbol_frequencies(c);
+ deflate_tally_item_list(c, block_length);
+
+ if (--num_passes_remaining == 0)
+ break;
+
+ /* At least one optimization pass remains; update the costs. */
+ deflate_make_huffman_codes(&c->freqs, &c->codes);
+ deflate_set_costs_from_codes(c, &c->codes.lens);
+ }
+}
+
+/*
+ * This is the "near-optimal" DEFLATE compressor. It computes the optimal
+ * representation of each DEFLATE block using a minimum-cost path search over
+ * the graph of possible match/literal choices for that block, assuming a
+ * certain cost for each Huffman symbol.
+ *
+ * For several reasons, the end result is not guaranteed to be optimal:
+ *
+ * - Nonoptimal choice of blocks
+ * - Heuristic limitations on which matches are actually considered
+ * - Symbol costs are unknown until the symbols have already been chosen
+ * (so iterative optimization must be used)
+ */
+static size_t
+deflate_compress_near_optimal(struct libdeflate_compressor * restrict c,
+ const u8 * restrict in, size_t in_nbytes,
+ u8 * restrict out, size_t out_nbytes_avail)
+{
+ const u8 *in_next = in;
+ const u8 *in_end = in_next + in_nbytes;
+ struct deflate_output_bitstream os;
+ const u8 *in_cur_base = in_next;
+ const u8 *in_next_slide = in_next + MIN(in_end - in_next, MATCHFINDER_WINDOW_SIZE);
+ unsigned max_len = DEFLATE_MAX_MATCH_LEN;
+ unsigned nice_len = MIN(c->nice_match_length, max_len);
+ u32 next_hashes[2] = {0, 0};
+
+ deflate_init_output(&os, out, out_nbytes_avail);
+ bt_matchfinder_init(&c->p.n.bt_mf);
+
+ do {
+ /* Starting a new DEFLATE block. */
+
+ struct lz_match *cache_ptr = c->p.n.match_cache;
+ const u8 * const in_block_begin = in_next;
+ const u8 * const in_max_block_end =
+ in_next + MIN(in_end - in_next, SOFT_MAX_BLOCK_LENGTH);
+ const u8 *next_observation = in_next;
+
+ init_block_split_stats(&c->split_stats);
+
+ /*
+ * Find matches until we decide to end the block. We end the
+ * block if any of the following is true:
+ *
+ * (1) Maximum block length has been reached
+ * (2) Match catch may overflow.
+ * (3) Block split heuristic says to split now.
+ */
+ do {
+ struct lz_match *matches;
+ unsigned best_len;
+
+ /* Slide the window forward if needed. */
+ if (in_next == in_next_slide) {
+ bt_matchfinder_slide_window(&c->p.n.bt_mf);
+ in_cur_base = in_next;
+ in_next_slide = in_next + MIN(in_end - in_next,
+ MATCHFINDER_WINDOW_SIZE);
+ }
+
+ /* Decrease the maximum and nice match lengths if we're
+ * approaching the end of the input buffer. */
+ if (unlikely(max_len > in_end - in_next)) {
+ max_len = in_end - in_next;
+ nice_len = MIN(nice_len, max_len);
+ }
+
+ /*
+ * Find matches with the current position using the
+ * binary tree matchfinder and save them in
+ * 'match_cache'.
+ *
+ * Note: the binary tree matchfinder is more suited for
+ * optimal parsing than the hash chain matchfinder. The
+ * reasons for this include:
+ *
+ * - The binary tree matchfinder can find more matches
+ * in the same number of steps.
+ * - One of the major advantages of hash chains is that
+ * skipping positions (not searching for matches at
+ * them) is faster; however, with optimal parsing we
+ * search for matches at almost all positions, so this
+ * advantage of hash chains is negated.
+ */
+ matches = cache_ptr;
+ best_len = 0;
+ if (likely(max_len >= BT_MATCHFINDER_REQUIRED_NBYTES)) {
+ cache_ptr = bt_matchfinder_get_matches(&c->p.n.bt_mf,
+ in_cur_base,
+ in_next - in_cur_base,
+ max_len,
+ nice_len,
+ c->max_search_depth,
+ next_hashes,
+ &best_len,
+ matches);
+ }
+
+ if (in_next >= next_observation) {
+ if (best_len >= 4) {
+ observe_match(&c->split_stats, best_len);
+ next_observation = in_next + best_len;
+ } else {
+ observe_literal(&c->split_stats, *in_next);
+ next_observation = in_next + 1;
+ }
+ }
+
+ cache_ptr->length = cache_ptr - matches;
+ cache_ptr->offset = *in_next;
+ in_next++;
+ cache_ptr++;
+
+ /*
+ * If there was a very long match found, don't cache any
+ * matches for the bytes covered by that match. This
+ * avoids degenerate behavior when compressing highly
+ * redundant data, where the number of matches can be
+ * very large.
+ *
+ * This heuristic doesn't actually hurt the compression
+ * ratio very much. If there's a long match, then the
+ * data must be highly compressible, so it doesn't
+ * matter much what we do.
+ */
+ if (best_len >= DEFLATE_MIN_MATCH_LEN && best_len >= nice_len) {
+ --best_len;
+ do {
+ if (in_next == in_next_slide) {
+ bt_matchfinder_slide_window(&c->p.n.bt_mf);
+ in_cur_base = in_next;
+ in_next_slide = in_next + MIN(in_end - in_next,
+ MATCHFINDER_WINDOW_SIZE);
+ }
+ if (unlikely(max_len > in_end - in_next)) {
+ max_len = in_end - in_next;
+ nice_len = MIN(nice_len, max_len);
+ }
+ if (max_len >= BT_MATCHFINDER_REQUIRED_NBYTES) {
+ bt_matchfinder_skip_position(&c->p.n.bt_mf,
+ in_cur_base,
+ in_next - in_cur_base,
+ nice_len,
+ c->max_search_depth,
+ next_hashes);
+ }
+ cache_ptr->length = 0;
+ cache_ptr->offset = *in_next;
+ in_next++;
+ cache_ptr++;
+ } while (--best_len);
+ }
+ } while (in_next < in_max_block_end &&
+ cache_ptr < &c->p.n.match_cache[CACHE_LENGTH] &&
+ !should_end_block(&c->split_stats, in_block_begin, in_next, in_end));
+
+ /* All the matches for this block have been cached. Now choose
+ * the sequence of items to output and flush the block. */
+ deflate_optimize_block(c, in_next - in_block_begin, cache_ptr,
+ in_block_begin == in);
+ deflate_flush_block(c, &os, in_block_begin, in_next - in_block_begin,
+ in_next == in_end, true);
+ } while (in_next != in_end);
+
+ return deflate_flush_output(&os);
+}
+
+#endif /* SUPPORT_NEAR_OPTIMAL_PARSING */
+
+/* Initialize c->offset_slot_fast. */
+static void
+deflate_init_offset_slot_fast(struct libdeflate_compressor *c)
+{
+ unsigned offset_slot;
+ unsigned offset;
+ unsigned offset_end;
+
+ for (offset_slot = 0;
+ offset_slot < ARRAY_LEN(deflate_offset_slot_base);
+ offset_slot++)
+ {
+ offset = deflate_offset_slot_base[offset_slot];
+ #if USE_FULL_OFFSET_SLOT_FAST
+ offset_end = offset + (1 << deflate_extra_offset_bits[offset_slot]);
+ do {
+ c->offset_slot_fast[offset] = offset_slot;
+ } while (++offset != offset_end);
+ #else
+ if (offset <= 256) {
+ offset_end = offset + (1 << deflate_extra_offset_bits[offset_slot]);
+ do {
+ c->offset_slot_fast[offset - 1] = offset_slot;
+ } while (++offset != offset_end);
+ } else {
+ offset_end = offset + (1 << deflate_extra_offset_bits[offset_slot]);
+ do {
+ c->offset_slot_fast[256 + ((offset - 1) >> 7)] = offset_slot;
+ } while ((offset += (1 << 7)) != offset_end);
+ }
+ #endif
+ }
+}
+
+LIBDEFLATEEXPORT struct libdeflate_compressor * LIBDEFLATEAPI
+libdeflate_alloc_compressor(int compression_level)
+{
+ struct libdeflate_compressor *c;
+ size_t size = offsetof(struct libdeflate_compressor, p);
+
+ if (compression_level < 0 || compression_level > 12)
+ return NULL;
+
+#if SUPPORT_NEAR_OPTIMAL_PARSING
+ if (compression_level >= 8)
+ size += sizeof(c->p.n);
+ else if (compression_level >= 1)
+ size += sizeof(c->p.g);
+#else
+ if (compression_level >= 1)
+ size += sizeof(c->p.g);
+#endif
+
+ c = libdeflate_aligned_malloc(MATCHFINDER_MEM_ALIGNMENT, size);
+ if (!c)
+ return NULL;
+
+ c->compression_level = compression_level;
+
+ /*
+ * The higher the compression level, the more we should bother trying to
+ * compress very small inputs.
+ */
+ c->min_size_to_compress = 56 - (compression_level * 4);
+
+ switch (compression_level) {
+ case 0:
+ c->impl = deflate_compress_none;
+ break;
+ case 1:
+ c->impl = deflate_compress_greedy;
+ c->max_search_depth = 2;
+ c->nice_match_length = 8;
+ break;
+ case 2:
+ c->impl = deflate_compress_greedy;
+ c->max_search_depth = 6;
+ c->nice_match_length = 10;
+ break;
+ case 3:
+ c->impl = deflate_compress_greedy;
+ c->max_search_depth = 12;
+ c->nice_match_length = 14;
+ break;
+ case 4:
+ c->impl = deflate_compress_greedy;
+ c->max_search_depth = 24;
+ c->nice_match_length = 24;
+ break;
+ case 5:
+ c->impl = deflate_compress_lazy;
+ c->max_search_depth = 20;
+ c->nice_match_length = 30;
+ break;
+ case 6:
+ c->impl = deflate_compress_lazy;
+ c->max_search_depth = 40;
+ c->nice_match_length = 65;
+ break;
+ case 7:
+ c->impl = deflate_compress_lazy;
+ c->max_search_depth = 100;
+ c->nice_match_length = 130;
+ break;
+#if SUPPORT_NEAR_OPTIMAL_PARSING
+ case 8:
+ c->impl = deflate_compress_near_optimal;
+ c->max_search_depth = 12;
+ c->nice_match_length = 20;
+ c->p.n.num_optim_passes = 1;
+ break;
+ case 9:
+ c->impl = deflate_compress_near_optimal;
+ c->max_search_depth = 16;
+ c->nice_match_length = 26;
+ c->p.n.num_optim_passes = 2;
+ break;
+ case 10:
+ c->impl = deflate_compress_near_optimal;
+ c->max_search_depth = 30;
+ c->nice_match_length = 50;
+ c->p.n.num_optim_passes = 2;
+ break;
+ case 11:
+ c->impl = deflate_compress_near_optimal;
+ c->max_search_depth = 60;
+ c->nice_match_length = 80;
+ c->p.n.num_optim_passes = 3;
+ break;
+ default:
+ c->impl = deflate_compress_near_optimal;
+ c->max_search_depth = 100;
+ c->nice_match_length = 133;
+ c->p.n.num_optim_passes = 4;
+ break;
+#else
+ case 8:
+ c->impl = deflate_compress_lazy;
+ c->max_search_depth = 150;
+ c->nice_match_length = 200;
+ break;
+ default:
+ c->impl = deflate_compress_lazy;
+ c->max_search_depth = 200;
+ c->nice_match_length = DEFLATE_MAX_MATCH_LEN;
+ break;
+#endif
+ }
+
+ deflate_init_offset_slot_fast(c);
+ deflate_init_static_codes(c);
+
+ return c;
+}
+
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_deflate_compress(struct libdeflate_compressor *c,
+ const void *in, size_t in_nbytes,
+ void *out, size_t out_nbytes_avail)
+{
+ if (unlikely(out_nbytes_avail < OUTPUT_END_PADDING))
+ return 0;
+
+ /* For extremely small inputs just use a single uncompressed block. */
+ if (unlikely(in_nbytes < c->min_size_to_compress)) {
+ struct deflate_output_bitstream os;
+ deflate_init_output(&os, out, out_nbytes_avail);
+ if (in_nbytes == 0)
+ in = &os; /* Avoid passing NULL to memcpy() */
+ deflate_write_uncompressed_block(&os, in, in_nbytes, true);
+ return deflate_flush_output(&os);
+ }
+
+ return (*c->impl)(c, in, in_nbytes, out, out_nbytes_avail);
+}
+
+LIBDEFLATEEXPORT void LIBDEFLATEAPI
+libdeflate_free_compressor(struct libdeflate_compressor *c)
+{
+ libdeflate_aligned_free(c);
+}
+
+unsigned int
+deflate_get_compression_level(struct libdeflate_compressor *c)
+{
+ return c->compression_level;
+}
+
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_deflate_compress_bound(struct libdeflate_compressor *c,
+ size_t in_nbytes)
+{
+ /*
+ * The worst case is all uncompressed blocks where one block has length
+ * <= MIN_BLOCK_LENGTH and the others have length MIN_BLOCK_LENGTH.
+ * Each uncompressed block has 5 bytes of overhead: 1 for BFINAL, BTYPE,
+ * and alignment to a byte boundary; 2 for LEN; and 2 for NLEN.
+ */
+ size_t max_num_blocks = MAX(DIV_ROUND_UP(in_nbytes, MIN_BLOCK_LENGTH), 1);
+ return (5 * max_num_blocks) + in_nbytes + 1 + OUTPUT_END_PADDING;
+}
diff --git a/ext/libdeflate/lib/deflate_compress.h b/ext/libdeflate/lib/deflate_compress.h
new file mode 100644
index 0000000000..d97d019ae7
--- /dev/null
+++ b/ext/libdeflate/lib/deflate_compress.h
@@ -0,0 +1,13 @@
+#ifndef LIB_DEFLATE_COMPRESS_H
+#define LIB_DEFLATE_COMPRESS_H
+
+#include "lib_common.h"
+
+/* DEFLATE compression is private to deflate_compress.c, but we do need to be
+ * able to query the compression level for zlib and gzip header generation. */
+
+struct libdeflate_compressor;
+
+unsigned int deflate_get_compression_level(struct libdeflate_compressor *c);
+
+#endif /* LIB_DEFLATE_COMPRESS_H */
diff --git a/ext/libdeflate/lib/deflate_constants.h b/ext/libdeflate/lib/deflate_constants.h
new file mode 100644
index 0000000000..a10b57dec0
--- /dev/null
+++ b/ext/libdeflate/lib/deflate_constants.h
@@ -0,0 +1,66 @@
+/*
+ * deflate_constants.h - constants for the DEFLATE compression format
+ */
+
+#ifndef LIB_DEFLATE_CONSTANTS_H
+#define LIB_DEFLATE_CONSTANTS_H
+
+/* Valid block types */
+#define DEFLATE_BLOCKTYPE_UNCOMPRESSED 0
+#define DEFLATE_BLOCKTYPE_STATIC_HUFFMAN 1
+#define DEFLATE_BLOCKTYPE_DYNAMIC_HUFFMAN 2
+
+/* Minimum and maximum supported match lengths (in bytes) */
+#define DEFLATE_MIN_MATCH_LEN 3
+#define DEFLATE_MAX_MATCH_LEN 258
+
+/* Minimum and maximum supported match offsets (in bytes) */
+#define DEFLATE_MIN_MATCH_OFFSET 1
+#define DEFLATE_MAX_MATCH_OFFSET 32768
+
+#define DEFLATE_MAX_WINDOW_SIZE 32768
+
+/* Number of symbols in each Huffman code. Note: for the literal/length
+ * and offset codes, these are actually the maximum values; a given block
+ * might use fewer symbols. */
+#define DEFLATE_NUM_PRECODE_SYMS 19
+#define DEFLATE_NUM_LITLEN_SYMS 288
+#define DEFLATE_NUM_OFFSET_SYMS 32
+
+/* The maximum number of symbols across all codes */
+#define DEFLATE_MAX_NUM_SYMS 288
+
+/* Division of symbols in the literal/length code */
+#define DEFLATE_NUM_LITERALS 256
+#define DEFLATE_END_OF_BLOCK 256
+#define DEFLATE_NUM_LEN_SYMS 31
+
+/* Maximum codeword length, in bits, within each Huffman code */
+#define DEFLATE_MAX_PRE_CODEWORD_LEN 7
+#define DEFLATE_MAX_LITLEN_CODEWORD_LEN 15
+#define DEFLATE_MAX_OFFSET_CODEWORD_LEN 15
+
+/* The maximum codeword length across all codes */
+#define DEFLATE_MAX_CODEWORD_LEN 15
+
+/* Maximum possible overrun when decoding codeword lengths */
+#define DEFLATE_MAX_LENS_OVERRUN 137
+
+/*
+ * Maximum number of extra bits that may be required to represent a match
+ * length or offset.
+ *
+ * TODO: are we going to have full DEFLATE64 support? If so, up to 16
+ * length bits must be supported.
+ */
+#define DEFLATE_MAX_EXTRA_LENGTH_BITS 5
+#define DEFLATE_MAX_EXTRA_OFFSET_BITS 14
+
+/* The maximum number of bits in which a match can be represented. This
+ * is the absolute worst case, which assumes the longest possible Huffman
+ * codewords and the maximum numbers of extra bits. */
+#define DEFLATE_MAX_MATCH_BITS \
+ (DEFLATE_MAX_LITLEN_CODEWORD_LEN + DEFLATE_MAX_EXTRA_LENGTH_BITS + \
+ DEFLATE_MAX_OFFSET_CODEWORD_LEN + DEFLATE_MAX_EXTRA_OFFSET_BITS)
+
+#endif /* LIB_DEFLATE_CONSTANTS_H */
diff --git a/ext/libdeflate/lib/hc_matchfinder.h b/ext/libdeflate/lib/hc_matchfinder.h
new file mode 100644
index 0000000000..becf844fa7
--- /dev/null
+++ b/ext/libdeflate/lib/hc_matchfinder.h
@@ -0,0 +1,407 @@
+/*
+ * hc_matchfinder.h - Lempel-Ziv matchfinding with a hash table of linked lists
+ *
+ * Originally public domain; changes after 2016-09-07 are copyrighted.
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * ---------------------------------------------------------------------------
+ *
+ * Algorithm
+ *
+ * This is a Hash Chains (hc) based matchfinder.
+ *
+ * The main data structure is a hash table where each hash bucket contains a
+ * linked list (or "chain") of sequences whose first 4 bytes share the same hash
+ * code. Each sequence is identified by its starting position in the input
+ * buffer.
+ *
+ * The algorithm processes the input buffer sequentially. At each byte
+ * position, the hash code of the first 4 bytes of the sequence beginning at
+ * that position (the sequence being matched against) is computed. This
+ * identifies the hash bucket to use for that position. Then, this hash
+ * bucket's linked list is searched for matches. Then, a new linked list node
+ * is created to represent the current sequence and is prepended to the list.
+ *
+ * This algorithm has several useful properties:
+ *
+ * - It only finds true Lempel-Ziv matches; i.e., those where the matching
+ * sequence occurs prior to the sequence being matched against.
+ *
+ * - The sequences in each linked list are always sorted by decreasing starting
+ * position. Therefore, the closest (smallest offset) matches are found
+ * first, which in many compression formats tend to be the cheapest to encode.
+ *
+ * - Although fast running time is not guaranteed due to the possibility of the
+ * lists getting very long, the worst degenerate behavior can be easily
+ * prevented by capping the number of nodes searched at each position.
+ *
+ * - If the compressor decides not to search for matches at a certain position,
+ * then that position can be quickly inserted without searching the list.
+ *
+ * - The algorithm is adaptable to sliding windows: just store the positions
+ * relative to a "base" value that is updated from time to time, and stop
+ * searching each list when the sequences get too far away.
+ *
+ * ----------------------------------------------------------------------------
+ *
+ * Optimizations
+ *
+ * The main hash table and chains handle length 4+ matches. Length 3 matches
+ * are handled by a separate hash table with no chains. This works well for
+ * typical "greedy" or "lazy"-style compressors, where length 3 matches are
+ * often only helpful if they have small offsets. Instead of searching a full
+ * chain for length 3+ matches, the algorithm just checks for one close length 3
+ * match, then focuses on finding length 4+ matches.
+ *
+ * The longest_match() and skip_positions() functions are inlined into the
+ * compressors that use them. This isn't just about saving the overhead of a
+ * function call. These functions are intended to be called from the inner
+ * loops of compressors, where giving the compiler more control over register
+ * allocation is very helpful. There is also significant benefit to be gained
+ * from allowing the CPU to predict branches independently at each call site.
+ * For example, "lazy"-style compressors can be written with two calls to
+ * longest_match(), each of which starts with a different 'best_len' and
+ * therefore has significantly different performance characteristics.
+ *
+ * Although any hash function can be used, a multiplicative hash is fast and
+ * works well.
+ *
+ * On some processors, it is significantly faster to extend matches by whole
+ * words (32 or 64 bits) instead of by individual bytes. For this to be the
+ * case, the processor must implement unaligned memory accesses efficiently and
+ * must have either a fast "find first set bit" instruction or a fast "find last
+ * set bit" instruction, depending on the processor's endianness.
+ *
+ * The code uses one loop for finding the first match and one loop for finding a
+ * longer match. Each of these loops is tuned for its respective task and in
+ * combination are faster than a single generalized loop that handles both
+ * tasks.
+ *
+ * The code also uses a tight inner loop that only compares the last and first
+ * bytes of a potential match. It is only when these bytes match that a full
+ * match extension is attempted.
+ *
+ * ----------------------------------------------------------------------------
+ */
+
+#include "matchfinder_common.h"
+
+#define HC_MATCHFINDER_HASH3_ORDER 15
+#define HC_MATCHFINDER_HASH4_ORDER 16
+
+#define HC_MATCHFINDER_TOTAL_HASH_SIZE \
+ (((1UL << HC_MATCHFINDER_HASH3_ORDER) + \
+ (1UL << HC_MATCHFINDER_HASH4_ORDER)) * sizeof(mf_pos_t))
+
+struct hc_matchfinder {
+
+ /* The hash table for finding length 3 matches */
+ mf_pos_t hash3_tab[1UL << HC_MATCHFINDER_HASH3_ORDER];
+
+ /* The hash table which contains the first nodes of the linked lists for
+ * finding length 4+ matches */
+ mf_pos_t hash4_tab[1UL << HC_MATCHFINDER_HASH4_ORDER];
+
+ /* The "next node" references for the linked lists. The "next node" of
+ * the node for the sequence with position 'pos' is 'next_tab[pos]'. */
+ mf_pos_t next_tab[MATCHFINDER_WINDOW_SIZE];
+
+}
+#ifdef _aligned_attribute
+ _aligned_attribute(MATCHFINDER_MEM_ALIGNMENT)
+#endif
+;
+
+/* Prepare the matchfinder for a new input buffer. */
+static forceinline void
+hc_matchfinder_init(struct hc_matchfinder *mf)
+{
+ STATIC_ASSERT(HC_MATCHFINDER_TOTAL_HASH_SIZE %
+ MATCHFINDER_SIZE_ALIGNMENT == 0);
+
+ matchfinder_init((mf_pos_t *)mf, HC_MATCHFINDER_TOTAL_HASH_SIZE);
+}
+
+static forceinline void
+hc_matchfinder_slide_window(struct hc_matchfinder *mf)
+{
+ STATIC_ASSERT(sizeof(*mf) % MATCHFINDER_SIZE_ALIGNMENT == 0);
+
+ matchfinder_rebase((mf_pos_t *)mf, sizeof(*mf));
+}
+
+/*
+ * Find the longest match longer than 'best_len' bytes.
+ *
+ * @mf
+ * The matchfinder structure.
+ * @in_base_p
+ * Location of a pointer which points to the place in the input data the
+ * matchfinder currently stores positions relative to. This may be updated
+ * by this function.
+ * @cur_pos
+ * The current position in the input buffer relative to @in_base (the
+ * position of the sequence being matched against).
+ * @best_len
+ * Require a match longer than this length.
+ * @max_len
+ * The maximum permissible match length at this position.
+ * @nice_len
+ * Stop searching if a match of at least this length is found.
+ * Must be <= @max_len.
+ * @max_search_depth
+ * Limit on the number of potential matches to consider. Must be >= 1.
+ * @next_hashes
+ * The precomputed hash codes for the sequence beginning at @in_next.
+ * These will be used and then updated with the precomputed hashcodes for
+ * the sequence beginning at @in_next + 1.
+ * @offset_ret
+ * If a match is found, its offset is returned in this location.
+ *
+ * Return the length of the match found, or 'best_len' if no match longer than
+ * 'best_len' was found.
+ */
+static forceinline u32
+hc_matchfinder_longest_match(struct hc_matchfinder * const restrict mf,
+ const u8 ** const restrict in_base_p,
+ const u8 * const restrict in_next,
+ u32 best_len,
+ const u32 max_len,
+ const u32 nice_len,
+ const u32 max_search_depth,
+ u32 * const restrict next_hashes,
+ u32 * const restrict offset_ret)
+{
+ u32 depth_remaining = max_search_depth;
+ const u8 *best_matchptr = in_next;
+ mf_pos_t cur_node3, cur_node4;
+ u32 hash3, hash4;
+ u32 next_hashseq;
+ u32 seq4;
+ const u8 *matchptr;
+ u32 len;
+ u32 cur_pos = in_next - *in_base_p;
+ const u8 *in_base;
+ mf_pos_t cutoff;
+
+ if (cur_pos == MATCHFINDER_WINDOW_SIZE) {
+ hc_matchfinder_slide_window(mf);
+ *in_base_p += MATCHFINDER_WINDOW_SIZE;
+ cur_pos = 0;
+ }
+
+ in_base = *in_base_p;
+ cutoff = cur_pos - MATCHFINDER_WINDOW_SIZE;
+
+ if (unlikely(max_len < 5)) /* can we read 4 bytes from 'in_next + 1'? */
+ goto out;
+
+ /* Get the precomputed hash codes. */
+ hash3 = next_hashes[0];
+ hash4 = next_hashes[1];
+
+ /* From the hash buckets, get the first node of each linked list. */
+ cur_node3 = mf->hash3_tab[hash3];
+ cur_node4 = mf->hash4_tab[hash4];
+
+ /* Update for length 3 matches. This replaces the singleton node in the
+ * 'hash3' bucket with the node for the current sequence. */
+ mf->hash3_tab[hash3] = cur_pos;
+
+ /* Update for length 4 matches. This prepends the node for the current
+ * sequence to the linked list in the 'hash4' bucket. */
+ mf->hash4_tab[hash4] = cur_pos;
+ mf->next_tab[cur_pos] = cur_node4;
+
+ /* Compute the next hash codes. */
+ next_hashseq = get_unaligned_le32(in_next + 1);
+ next_hashes[0] = lz_hash(next_hashseq & 0xFFFFFF, HC_MATCHFINDER_HASH3_ORDER);
+ next_hashes[1] = lz_hash(next_hashseq, HC_MATCHFINDER_HASH4_ORDER);
+ prefetchw(&mf->hash3_tab[next_hashes[0]]);
+ prefetchw(&mf->hash4_tab[next_hashes[1]]);
+
+ if (best_len < 4) { /* No match of length >= 4 found yet? */
+
+ /* Check for a length 3 match if needed. */
+
+ if (cur_node3 <= cutoff)
+ goto out;
+
+ seq4 = load_u32_unaligned(in_next);
+
+ if (best_len < 3) {
+ matchptr = &in_base[cur_node3];
+ if (load_u24_unaligned(matchptr) == loaded_u32_to_u24(seq4)) {
+ best_len = 3;
+ best_matchptr = matchptr;
+ }
+ }
+
+ /* Check for a length 4 match. */
+
+ if (cur_node4 <= cutoff)
+ goto out;
+
+ for (;;) {
+ /* No length 4 match found yet. Check the first 4 bytes. */
+ matchptr = &in_base[cur_node4];
+
+ if (load_u32_unaligned(matchptr) == seq4)
+ break;
+
+ /* The first 4 bytes did not match. Keep trying. */
+ cur_node4 = mf->next_tab[cur_node4 & (MATCHFINDER_WINDOW_SIZE - 1)];
+ if (cur_node4 <= cutoff || !--depth_remaining)
+ goto out;
+ }
+
+ /* Found a match of length >= 4. Extend it to its full length. */
+ best_matchptr = matchptr;
+ best_len = lz_extend(in_next, best_matchptr, 4, max_len);
+ if (best_len >= nice_len)
+ goto out;
+ cur_node4 = mf->next_tab[cur_node4 & (MATCHFINDER_WINDOW_SIZE - 1)];
+ if (cur_node4 <= cutoff || !--depth_remaining)
+ goto out;
+ } else {
+ if (cur_node4 <= cutoff || best_len >= nice_len)
+ goto out;
+ }
+
+ /* Check for matches of length >= 5. */
+
+ for (;;) {
+ for (;;) {
+ matchptr = &in_base[cur_node4];
+
+ /* Already found a length 4 match. Try for a longer
+ * match; start by checking either the last 4 bytes and
+ * the first 4 bytes, or the last byte. (The last byte,
+ * the one which would extend the match length by 1, is
+ * the most important.) */
+ #if UNALIGNED_ACCESS_IS_FAST
+ if ((load_u32_unaligned(matchptr + best_len - 3) ==
+ load_u32_unaligned(in_next + best_len - 3)) &&
+ (load_u32_unaligned(matchptr) ==
+ load_u32_unaligned(in_next)))
+ #else
+ if (matchptr[best_len] == in_next[best_len])
+ #endif
+ break;
+
+ /* Continue to the next node in the list. */
+ cur_node4 = mf->next_tab[cur_node4 & (MATCHFINDER_WINDOW_SIZE - 1)];
+ if (cur_node4 <= cutoff || !--depth_remaining)
+ goto out;
+ }
+
+ #if UNALIGNED_ACCESS_IS_FAST
+ len = 4;
+ #else
+ len = 0;
+ #endif
+ len = lz_extend(in_next, matchptr, len, max_len);
+ if (len > best_len) {
+ /* This is the new longest match. */
+ best_len = len;
+ best_matchptr = matchptr;
+ if (best_len >= nice_len)
+ goto out;
+ }
+
+ /* Continue to the next node in the list. */
+ cur_node4 = mf->next_tab[cur_node4 & (MATCHFINDER_WINDOW_SIZE - 1)];
+ if (cur_node4 <= cutoff || !--depth_remaining)
+ goto out;
+ }
+out:
+ *offset_ret = in_next - best_matchptr;
+ return best_len;
+}
+
+/*
+ * Advance the matchfinder, but don't search for matches.
+ *
+ * @mf
+ * The matchfinder structure.
+ * @in_base_p
+ * Location of a pointer which points to the place in the input data the
+ * matchfinder currently stores positions relative to. This may be updated
+ * by this function.
+ * @cur_pos
+ * The current position in the input buffer relative to @in_base.
+ * @end_pos
+ * The end position of the input buffer, relative to @in_base.
+ * @next_hashes
+ * The precomputed hash codes for the sequence beginning at @in_next.
+ * These will be used and then updated with the precomputed hashcodes for
+ * the sequence beginning at @in_next + @count.
+ * @count
+ * The number of bytes to advance. Must be > 0.
+ *
+ * Returns @in_next + @count.
+ */
+static forceinline const u8 *
+hc_matchfinder_skip_positions(struct hc_matchfinder * const restrict mf,
+ const u8 ** const restrict in_base_p,
+ const u8 *in_next,
+ const u8 * const in_end,
+ const u32 count,
+ u32 * const restrict next_hashes)
+{
+ u32 cur_pos;
+ u32 hash3, hash4;
+ u32 next_hashseq;
+ u32 remaining = count;
+
+ if (unlikely(count + 5 > in_end - in_next))
+ return &in_next[count];
+
+ cur_pos = in_next - *in_base_p;
+ hash3 = next_hashes[0];
+ hash4 = next_hashes[1];
+ do {
+ if (cur_pos == MATCHFINDER_WINDOW_SIZE) {
+ hc_matchfinder_slide_window(mf);
+ *in_base_p += MATCHFINDER_WINDOW_SIZE;
+ cur_pos = 0;
+ }
+ mf->hash3_tab[hash3] = cur_pos;
+ mf->next_tab[cur_pos] = mf->hash4_tab[hash4];
+ mf->hash4_tab[hash4] = cur_pos;
+
+ next_hashseq = get_unaligned_le32(++in_next);
+ hash3 = lz_hash(next_hashseq & 0xFFFFFF, HC_MATCHFINDER_HASH3_ORDER);
+ hash4 = lz_hash(next_hashseq, HC_MATCHFINDER_HASH4_ORDER);
+ cur_pos++;
+ } while (--remaining);
+
+ prefetchw(&mf->hash3_tab[hash3]);
+ prefetchw(&mf->hash4_tab[hash4]);
+ next_hashes[0] = hash3;
+ next_hashes[1] = hash4;
+
+ return in_next;
+}
diff --git a/ext/libdeflate/lib/lib_common.h b/ext/libdeflate/lib/lib_common.h
new file mode 100644
index 0000000000..2eea56c729
--- /dev/null
+++ b/ext/libdeflate/lib/lib_common.h
@@ -0,0 +1,67 @@
+/*
+ * lib_common.h - internal header included by all library code
+ */
+
+#ifndef LIB_LIB_COMMON_H
+#define LIB_LIB_COMMON_H
+
+#ifdef LIBDEFLATE_H
+# error "lib_common.h must always be included before libdeflate.h"
+ /* because BUILDING_LIBDEFLATE must be set first */
+#endif
+
+#define BUILDING_LIBDEFLATE
+
+#include "../common/common_defs.h"
+
+/*
+ * Prefix with "_libdeflate_" all global symbols which are not part of the API
+ * and don't already have a "libdeflate" prefix. This avoids exposing overly
+ * generic names when libdeflate is built as a static library.
+ *
+ * Note that the chosen prefix is not really important and can be changed
+ * without breaking library users. It was just chosen so that the resulting
+ * symbol names are unlikely to conflict with those from any other software.
+ * Also note that this fixup has no useful effect when libdeflate is built as a
+ * shared library, since these symbols are not exported.
+ */
+#define SYM_FIXUP(sym) _libdeflate_##sym
+#define deflate_get_compression_level SYM_FIXUP(deflate_get_compression_level)
+#define _cpu_features SYM_FIXUP(_cpu_features)
+#define setup_cpu_features SYM_FIXUP(setup_cpu_features)
+
+void *libdeflate_malloc(size_t size);
+void libdeflate_free(void *ptr);
+
+void *libdeflate_aligned_malloc(size_t alignment, size_t size);
+void libdeflate_aligned_free(void *ptr);
+
+#ifdef FREESTANDING
+/*
+ * With -ffreestanding, may be missing, and we must provide
+ * implementations of memset(), memcpy(), memmove(), and memcmp().
+ * See https://gcc.gnu.org/onlinedocs/gcc/Standards.html
+ *
+ * Also, -ffreestanding disables interpreting calls to these functions as
+ * built-ins. E.g., calling memcpy(&v, p, WORDBYTES) will make a function call,
+ * not be optimized to a single load instruction. For performance reasons we
+ * don't want that. So, declare these functions as macros that expand to the
+ * corresponding built-ins. This approach is recommended in the gcc man page.
+ * We still need the actual function definitions in case gcc calls them.
+ */
+void *memset(void *s, int c, size_t n);
+#define memset(s, c, n) __builtin_memset((s), (c), (n))
+
+void *memcpy(void *dest, const void *src, size_t n);
+#define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
+
+void *memmove(void *dest, const void *src, size_t n);
+#define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
+
+int memcmp(const void *s1, const void *s2, size_t n);
+#define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
+#else
+#include
+#endif
+
+#endif /* LIB_LIB_COMMON_H */
diff --git a/ext/libdeflate/lib/matchfinder_common.h b/ext/libdeflate/lib/matchfinder_common.h
new file mode 100644
index 0000000000..49ff3343e1
--- /dev/null
+++ b/ext/libdeflate/lib/matchfinder_common.h
@@ -0,0 +1,176 @@
+/*
+ * matchfinder_common.h - common code for Lempel-Ziv matchfinding
+ */
+
+#ifndef LIB_MATCHFINDER_COMMON_H
+#define LIB_MATCHFINDER_COMMON_H
+
+#include "lib_common.h"
+#include "unaligned.h"
+
+#ifndef MATCHFINDER_WINDOW_ORDER
+# error "MATCHFINDER_WINDOW_ORDER must be defined!"
+#endif
+
+#define MATCHFINDER_WINDOW_SIZE (1UL << MATCHFINDER_WINDOW_ORDER)
+
+typedef s16 mf_pos_t;
+
+#define MATCHFINDER_INITVAL ((mf_pos_t)-MATCHFINDER_WINDOW_SIZE)
+
+/*
+ * Required alignment of the matchfinder buffer pointer and size. The values
+ * here come from the AVX-2 implementation, which is the worst case.
+ */
+#define MATCHFINDER_MEM_ALIGNMENT 32
+#define MATCHFINDER_SIZE_ALIGNMENT 128
+
+#undef matchfinder_init
+#undef matchfinder_rebase
+#ifdef _aligned_attribute
+# if defined(__arm__) || defined(__aarch64__)
+# include "arm/matchfinder_impl.h"
+# elif defined(__i386__) || defined(__x86_64__)
+# include "x86/matchfinder_impl.h"
+# endif
+#endif
+
+/*
+ * Initialize the hash table portion of the matchfinder.
+ *
+ * Essentially, this is an optimized memset().
+ *
+ * 'data' must be aligned to a MATCHFINDER_MEM_ALIGNMENT boundary, and
+ * 'size' must be a multiple of MATCHFINDER_SIZE_ALIGNMENT.
+ */
+#ifndef matchfinder_init
+static forceinline void
+matchfinder_init(mf_pos_t *data, size_t size)
+{
+ size_t num_entries = size / sizeof(*data);
+ size_t i;
+
+ for (i = 0; i < num_entries; i++)
+ data[i] = MATCHFINDER_INITVAL;
+}
+#endif
+
+/*
+ * Slide the matchfinder by WINDOW_SIZE bytes.
+ *
+ * This must be called just after each WINDOW_SIZE bytes have been run through
+ * the matchfinder.
+ *
+ * This will subtract WINDOW_SIZE bytes from each entry in the array specified.
+ * The effect is that all entries are updated to be relative to the current
+ * position, rather than the position WINDOW_SIZE bytes prior.
+ *
+ * Underflow is detected and replaced with signed saturation. This ensures that
+ * once the sliding window has passed over a position, that position forever
+ * remains out of bounds.
+ *
+ * The array passed in must contain all matchfinder data that is
+ * position-relative. Concretely, this will include the hash table as well as
+ * the table of positions that is used to link together the sequences in each
+ * hash bucket. Note that in the latter table, the links are 1-ary in the case
+ * of "hash chains", and 2-ary in the case of "binary trees". In either case,
+ * the links need to be rebased in the same way.
+ *
+ * 'data' must be aligned to a MATCHFINDER_MEM_ALIGNMENT boundary, and
+ * 'size' must be a multiple of MATCHFINDER_SIZE_ALIGNMENT.
+ */
+#ifndef matchfinder_rebase
+static forceinline void
+matchfinder_rebase(mf_pos_t *data, size_t size)
+{
+ size_t num_entries = size / sizeof(*data);
+ size_t i;
+
+ if (MATCHFINDER_WINDOW_SIZE == 32768) {
+ /* Branchless version for 32768 byte windows. If the value was
+ * already negative, clear all bits except the sign bit; this
+ * changes the value to -32768. Otherwise, set the sign bit;
+ * this is equivalent to subtracting 32768. */
+ for (i = 0; i < num_entries; i++) {
+ u16 v = data[i];
+ u16 sign_bit = v & 0x8000;
+ v &= sign_bit - ((sign_bit >> 15) ^ 1);
+ v |= 0x8000;
+ data[i] = v;
+ }
+ return;
+ }
+
+ for (i = 0; i < num_entries; i++) {
+ if (data[i] >= 0)
+ data[i] -= (mf_pos_t)-MATCHFINDER_WINDOW_SIZE;
+ else
+ data[i] = (mf_pos_t)-MATCHFINDER_WINDOW_SIZE;
+ }
+}
+#endif
+
+/*
+ * The hash function: given a sequence prefix held in the low-order bits of a
+ * 32-bit value, multiply by a carefully-chosen large constant. Discard any
+ * bits of the product that don't fit in a 32-bit value, but take the
+ * next-highest @num_bits bits of the product as the hash value, as those have
+ * the most randomness.
+ */
+static forceinline u32
+lz_hash(u32 seq, unsigned num_bits)
+{
+ return (u32)(seq * 0x1E35A7BD) >> (32 - num_bits);
+}
+
+/*
+ * Return the number of bytes at @matchptr that match the bytes at @strptr, up
+ * to a maximum of @max_len. Initially, @start_len bytes are matched.
+ */
+static forceinline unsigned
+lz_extend(const u8 * const strptr, const u8 * const matchptr,
+ const unsigned start_len, const unsigned max_len)
+{
+ unsigned len = start_len;
+ machine_word_t v_word;
+
+ if (UNALIGNED_ACCESS_IS_FAST) {
+
+ if (likely(max_len - len >= 4 * WORDBYTES)) {
+
+ #define COMPARE_WORD_STEP \
+ v_word = load_word_unaligned(&matchptr[len]) ^ \
+ load_word_unaligned(&strptr[len]); \
+ if (v_word != 0) \
+ goto word_differs; \
+ len += WORDBYTES; \
+
+ COMPARE_WORD_STEP
+ COMPARE_WORD_STEP
+ COMPARE_WORD_STEP
+ COMPARE_WORD_STEP
+ #undef COMPARE_WORD_STEP
+ }
+
+ while (len + WORDBYTES <= max_len) {
+ v_word = load_word_unaligned(&matchptr[len]) ^
+ load_word_unaligned(&strptr[len]);
+ if (v_word != 0)
+ goto word_differs;
+ len += WORDBYTES;
+ }
+ }
+
+ while (len < max_len && matchptr[len] == strptr[len])
+ len++;
+ return len;
+
+word_differs:
+ if (CPU_IS_LITTLE_ENDIAN())
+ len += (bsfw(v_word) >> 3);
+ else
+ len += (WORDBITS - 1 - bsrw(v_word)) >> 3;
+ return len;
+}
+
+#endif /* LIB_MATCHFINDER_COMMON_H */
diff --git a/ext/libdeflate/lib/unaligned.h b/ext/libdeflate/lib/unaligned.h
new file mode 100644
index 0000000000..bb48bf8283
--- /dev/null
+++ b/ext/libdeflate/lib/unaligned.h
@@ -0,0 +1,228 @@
+/*
+ * unaligned.h - inline functions for unaligned memory accesses
+ */
+
+#ifndef LIB_UNALIGNED_H
+#define LIB_UNALIGNED_H
+
+#include "lib_common.h"
+
+/***** Unaligned loads and stores without endianness conversion *****/
+
+/*
+ * memcpy() is portable, and it usually gets optimized appropriately by modern
+ * compilers. I.e., each memcpy() of 1, 2, 4, or WORDBYTES bytes gets compiled
+ * to a load or store instruction, not to an actual function call.
+ *
+ * We no longer use the "packed struct" approach, as that is nonstandard, has
+ * unclear semantics, and doesn't receive enough testing
+ * (see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=94994).
+ *
+ * arm32 with __ARM_FEATURE_UNALIGNED in gcc 5 and earlier is a known exception
+ * where memcpy() generates inefficient code
+ * (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67366). However, we no longer
+ * consider that one case important enough to maintain different code for.
+ * If you run into it, please just use a newer version of gcc (or use clang).
+ */
+
+#define DEFINE_UNALIGNED_TYPE(type) \
+static forceinline type \
+load_##type##_unaligned(const void *p) \
+{ \
+ type v; \
+ memcpy(&v, p, sizeof(v)); \
+ return v; \
+} \
+ \
+static forceinline void \
+store_##type##_unaligned(type v, void *p) \
+{ \
+ memcpy(p, &v, sizeof(v)); \
+}
+
+DEFINE_UNALIGNED_TYPE(u16)
+DEFINE_UNALIGNED_TYPE(u32)
+DEFINE_UNALIGNED_TYPE(u64)
+DEFINE_UNALIGNED_TYPE(machine_word_t)
+
+#define load_word_unaligned load_machine_word_t_unaligned
+#define store_word_unaligned store_machine_word_t_unaligned
+
+/***** Unaligned loads with endianness conversion *****/
+
+static forceinline u16
+get_unaligned_le16(const u8 *p)
+{
+ if (UNALIGNED_ACCESS_IS_FAST)
+ return le16_bswap(load_u16_unaligned(p));
+ else
+ return ((u16)p[1] << 8) | p[0];
+}
+
+static forceinline u16
+get_unaligned_be16(const u8 *p)
+{
+ if (UNALIGNED_ACCESS_IS_FAST)
+ return be16_bswap(load_u16_unaligned(p));
+ else
+ return ((u16)p[0] << 8) | p[1];
+}
+
+static forceinline u32
+get_unaligned_le32(const u8 *p)
+{
+ if (UNALIGNED_ACCESS_IS_FAST)
+ return le32_bswap(load_u32_unaligned(p));
+ else
+ return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
+ ((u32)p[1] << 8) | p[0];
+}
+
+static forceinline u32
+get_unaligned_be32(const u8 *p)
+{
+ if (UNALIGNED_ACCESS_IS_FAST)
+ return be32_bswap(load_u32_unaligned(p));
+ else
+ return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
+ ((u32)p[2] << 8) | p[3];
+}
+
+static forceinline u64
+get_unaligned_le64(const u8 *p)
+{
+ if (UNALIGNED_ACCESS_IS_FAST)
+ return le64_bswap(load_u64_unaligned(p));
+ else
+ return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
+ ((u64)p[5] << 40) | ((u64)p[4] << 32) |
+ ((u64)p[3] << 24) | ((u64)p[2] << 16) |
+ ((u64)p[1] << 8) | p[0];
+}
+
+static forceinline machine_word_t
+get_unaligned_leword(const u8 *p)
+{
+ STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
+ if (WORDBITS == 32)
+ return get_unaligned_le32(p);
+ else
+ return get_unaligned_le64(p);
+}
+
+/***** Unaligned stores with endianness conversion *****/
+
+static forceinline void
+put_unaligned_le16(u16 v, u8 *p)
+{
+ if (UNALIGNED_ACCESS_IS_FAST) {
+ store_u16_unaligned(le16_bswap(v), p);
+ } else {
+ p[0] = (u8)(v >> 0);
+ p[1] = (u8)(v >> 8);
+ }
+}
+
+static forceinline void
+put_unaligned_be16(u16 v, u8 *p)
+{
+ if (UNALIGNED_ACCESS_IS_FAST) {
+ store_u16_unaligned(be16_bswap(v), p);
+ } else {
+ p[0] = (u8)(v >> 8);
+ p[1] = (u8)(v >> 0);
+ }
+}
+
+static forceinline void
+put_unaligned_le32(u32 v, u8 *p)
+{
+ if (UNALIGNED_ACCESS_IS_FAST) {
+ store_u32_unaligned(le32_bswap(v), p);
+ } else {
+ p[0] = (u8)(v >> 0);
+ p[1] = (u8)(v >> 8);
+ p[2] = (u8)(v >> 16);
+ p[3] = (u8)(v >> 24);
+ }
+}
+
+static forceinline void
+put_unaligned_be32(u32 v, u8 *p)
+{
+ if (UNALIGNED_ACCESS_IS_FAST) {
+ store_u32_unaligned(be32_bswap(v), p);
+ } else {
+ p[0] = (u8)(v >> 24);
+ p[1] = (u8)(v >> 16);
+ p[2] = (u8)(v >> 8);
+ p[3] = (u8)(v >> 0);
+ }
+}
+
+static forceinline void
+put_unaligned_le64(u64 v, u8 *p)
+{
+ if (UNALIGNED_ACCESS_IS_FAST) {
+ store_u64_unaligned(le64_bswap(v), p);
+ } else {
+ p[0] = (u8)(v >> 0);
+ p[1] = (u8)(v >> 8);
+ p[2] = (u8)(v >> 16);
+ p[3] = (u8)(v >> 24);
+ p[4] = (u8)(v >> 32);
+ p[5] = (u8)(v >> 40);
+ p[6] = (u8)(v >> 48);
+ p[7] = (u8)(v >> 56);
+ }
+}
+
+static forceinline void
+put_unaligned_leword(machine_word_t v, u8 *p)
+{
+ STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
+ if (WORDBITS == 32)
+ put_unaligned_le32(v, p);
+ else
+ put_unaligned_le64(v, p);
+}
+
+/***** 24-bit loads *****/
+
+/*
+ * Given a 32-bit value that was loaded with the platform's native endianness,
+ * return a 32-bit value whose high-order 8 bits are 0 and whose low-order 24
+ * bits contain the first 3 bytes, arranged in octets in a platform-dependent
+ * order, at the memory location from which the input 32-bit value was loaded.
+ */
+static forceinline u32
+loaded_u32_to_u24(u32 v)
+{
+ if (CPU_IS_LITTLE_ENDIAN())
+ return v & 0xFFFFFF;
+ else
+ return v >> 8;
+}
+
+/*
+ * Load the next 3 bytes from the memory location @p into the 24 low-order bits
+ * of a 32-bit value. The order in which the 3 bytes will be arranged as octets
+ * in the 24 bits is platform-dependent. At least LOAD_U24_REQUIRED_NBYTES
+ * bytes must be available at @p; note that this may be more than 3.
+ */
+static forceinline u32
+load_u24_unaligned(const u8 *p)
+{
+#if UNALIGNED_ACCESS_IS_FAST
+# define LOAD_U24_REQUIRED_NBYTES 4
+ return loaded_u32_to_u24(load_u32_unaligned(p));
+#else
+# define LOAD_U24_REQUIRED_NBYTES 3
+ if (CPU_IS_LITTLE_ENDIAN())
+ return ((u32)p[0] << 0) | ((u32)p[1] << 8) | ((u32)p[2] << 16);
+ else
+ return ((u32)p[2] << 0) | ((u32)p[1] << 8) | ((u32)p[0] << 16);
+#endif
+}
+
+#endif /* LIB_UNALIGNED_H */
diff --git a/ext/libdeflate/lib/utils.c b/ext/libdeflate/lib/utils.c
new file mode 100644
index 0000000000..ed76046368
--- /dev/null
+++ b/ext/libdeflate/lib/utils.c
@@ -0,0 +1,138 @@
+/*
+ * utils.c - utility functions for libdeflate
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "lib_common.h"
+
+#include "../libdeflate.h"
+
+#ifdef FREESTANDING
+# define malloc NULL
+# define free NULL
+#else
+# include
+#endif
+
+static void *(*libdeflate_malloc_func)(size_t) = malloc;
+static void (*libdeflate_free_func)(void *) = free;
+
+void *
+libdeflate_malloc(size_t size)
+{
+ return (*libdeflate_malloc_func)(size);
+}
+
+void
+libdeflate_free(void *ptr)
+{
+ (*libdeflate_free_func)(ptr);
+}
+
+void *
+libdeflate_aligned_malloc(size_t alignment, size_t size)
+{
+ void *ptr = libdeflate_malloc(sizeof(void *) + alignment - 1 + size);
+ if (ptr) {
+ void *orig_ptr = ptr;
+ ptr = (void *)ALIGN((uintptr_t)ptr + sizeof(void *), alignment);
+ ((void **)ptr)[-1] = orig_ptr;
+ }
+ return ptr;
+}
+
+void
+libdeflate_aligned_free(void *ptr)
+{
+ if (ptr)
+ libdeflate_free(((void **)ptr)[-1]);
+}
+
+LIBDEFLATEEXPORT void LIBDEFLATEAPI
+libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
+ void (*free_func)(void *))
+{
+ libdeflate_malloc_func = malloc_func;
+ libdeflate_free_func = free_func;
+}
+
+/*
+ * Implementations of libc functions for freestanding library builds.
+ * Normal library builds don't use these. Not optimized yet; usually the
+ * compiler expands these functions and doesn't actually call them anyway.
+ */
+#ifdef FREESTANDING
+#undef memset
+void *memset(void *s, int c, size_t n)
+{
+ u8 *p = s;
+ size_t i;
+
+ for (i = 0; i < n; i++)
+ p[i] = c;
+ return s;
+}
+
+#undef memcpy
+void *memcpy(void *dest, const void *src, size_t n)
+{
+ u8 *d = dest;
+ const u8 *s = src;
+ size_t i;
+
+ for (i = 0; i < n; i++)
+ d[i] = s[i];
+ return dest;
+}
+
+#undef memmove
+void *memmove(void *dest, const void *src, size_t n)
+{
+ u8 *d = dest;
+ const u8 *s = src;
+ size_t i;
+
+ if (d <= s)
+ return memcpy(d, s, n);
+
+ for (i = n; i > 0; i--)
+ d[i - 1] = s[i - 1];
+ return dest;
+}
+
+#undef memcmp
+int memcmp(const void *s1, const void *s2, size_t n)
+{
+ const u8 *p1 = s1;
+ const u8 *p2 = s2;
+ size_t i;
+
+ for (i = 0; i < n; i++) {
+ if (p1[i] != p2[i])
+ return (int)p1[i] - (int)p2[i];
+ }
+ return 0;
+}
+#endif /* FREESTANDING */
diff --git a/ext/libdeflate/lib/x86/matchfinder_impl.h b/ext/libdeflate/lib/x86/matchfinder_impl.h
new file mode 100644
index 0000000000..22a6c82da3
--- /dev/null
+++ b/ext/libdeflate/lib/x86/matchfinder_impl.h
@@ -0,0 +1,117 @@
+/*
+ * x86/matchfinder_impl.h - x86 implementations of matchfinder functions
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifdef __AVX2__
+# include
+static forceinline void
+matchfinder_init_avx2(mf_pos_t *data, size_t size)
+{
+ __m256i *p = (__m256i *)data;
+ __m256i v = _mm256_set1_epi16(MATCHFINDER_INITVAL);
+
+ STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
+ STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
+ STATIC_ASSERT(sizeof(mf_pos_t) == 2);
+
+ do {
+ p[0] = v;
+ p[1] = v;
+ p[2] = v;
+ p[3] = v;
+ p += 4;
+ size -= 4 * sizeof(*p);
+ } while (size != 0);
+}
+#define matchfinder_init matchfinder_init_avx2
+
+static forceinline void
+matchfinder_rebase_avx2(mf_pos_t *data, size_t size)
+{
+ __m256i *p = (__m256i *)data;
+ __m256i v = _mm256_set1_epi16((u16)-MATCHFINDER_WINDOW_SIZE);
+
+ STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
+ STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
+ STATIC_ASSERT(sizeof(mf_pos_t) == 2);
+
+ do {
+ /* PADDSW: Add Packed Signed Integers With Signed Saturation */
+ p[0] = _mm256_adds_epi16(p[0], v);
+ p[1] = _mm256_adds_epi16(p[1], v);
+ p[2] = _mm256_adds_epi16(p[2], v);
+ p[3] = _mm256_adds_epi16(p[3], v);
+ p += 4;
+ size -= 4 * sizeof(*p);
+ } while (size != 0);
+}
+#define matchfinder_rebase matchfinder_rebase_avx2
+
+#elif defined(__SSE2__)
+# include
+static forceinline void
+matchfinder_init_sse2(mf_pos_t *data, size_t size)
+{
+ __m128i *p = (__m128i *)data;
+ __m128i v = _mm_set1_epi16(MATCHFINDER_INITVAL);
+
+ STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
+ STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
+ STATIC_ASSERT(sizeof(mf_pos_t) == 2);
+
+ do {
+ p[0] = v;
+ p[1] = v;
+ p[2] = v;
+ p[3] = v;
+ p += 4;
+ size -= 4 * sizeof(*p);
+ } while (size != 0);
+}
+#define matchfinder_init matchfinder_init_sse2
+
+static forceinline void
+matchfinder_rebase_sse2(mf_pos_t *data, size_t size)
+{
+ __m128i *p = (__m128i *)data;
+ __m128i v = _mm_set1_epi16((u16)-MATCHFINDER_WINDOW_SIZE);
+
+ STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
+ STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
+ STATIC_ASSERT(sizeof(mf_pos_t) == 2);
+
+ do {
+ /* PADDSW: Add Packed Signed Integers With Signed Saturation */
+ p[0] = _mm_adds_epi16(p[0], v);
+ p[1] = _mm_adds_epi16(p[1], v);
+ p[2] = _mm_adds_epi16(p[2], v);
+ p[3] = _mm_adds_epi16(p[3], v);
+ p += 4;
+ size -= 4 * sizeof(*p);
+ } while (size != 0);
+}
+#define matchfinder_rebase matchfinder_rebase_sse2
+#endif /* __SSE2__ */
diff --git a/ext/libdeflate/libdeflate.h b/ext/libdeflate/libdeflate.h
new file mode 100644
index 0000000000..cbcda04202
--- /dev/null
+++ b/ext/libdeflate/libdeflate.h
@@ -0,0 +1,366 @@
+/*
+ * libdeflate.h - public header for libdeflate
+ */
+
+#ifndef LIBDEFLATE_H
+#define LIBDEFLATE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define LIBDEFLATE_VERSION_MAJOR 1
+#define LIBDEFLATE_VERSION_MINOR 7
+#define LIBDEFLATE_VERSION_STRING "1.7"
+
+#include
+#include
+
+/*
+ * On Windows, if you want to link to the DLL version of libdeflate, then
+ * #define LIBDEFLATE_DLL. Note that the calling convention is "stdcall".
+ */
+#ifdef LIBDEFLATE_DLL
+# ifdef BUILDING_LIBDEFLATE
+# define LIBDEFLATEEXPORT LIBEXPORT
+# elif defined(_WIN32) || defined(__CYGWIN__)
+# define LIBDEFLATEEXPORT __declspec(dllimport)
+# endif
+#endif
+#ifndef LIBDEFLATEEXPORT
+# define LIBDEFLATEEXPORT
+#endif
+
+#if defined(_WIN32) && !defined(_WIN64)
+# define LIBDEFLATEAPI_ABI __stdcall
+#else
+# define LIBDEFLATEAPI_ABI
+#endif
+
+#if defined(BUILDING_LIBDEFLATE) && defined(__GNUC__) && \
+ defined(_WIN32) && !defined(_WIN64)
+ /*
+ * On 32-bit Windows, gcc assumes 16-byte stack alignment but MSVC only 4.
+ * Realign the stack when entering libdeflate to avoid crashing in SSE/AVX
+ * code when called from an MSVC-compiled application.
+ */
+# define LIBDEFLATEAPI_STACKALIGN __attribute__((force_align_arg_pointer))
+#else
+# define LIBDEFLATEAPI_STACKALIGN
+#endif
+
+#define LIBDEFLATEAPI LIBDEFLATEAPI_ABI LIBDEFLATEAPI_STACKALIGN
+
+/* ========================================================================== */
+/* Compression */
+/* ========================================================================== */
+
+struct libdeflate_compressor;
+
+/*
+ * libdeflate_alloc_compressor() allocates a new compressor that supports
+ * DEFLATE, zlib, and gzip compression. 'compression_level' is the compression
+ * level on a zlib-like scale but with a higher maximum value (1 = fastest, 6 =
+ * medium/default, 9 = slow, 12 = slowest). Level 0 is also supported and means
+ * "no compression", specifically "create a valid stream, but only emit
+ * uncompressed blocks" (this will expand the data slightly).
+ *
+ * The return value is a pointer to the new compressor, or NULL if out of memory
+ * or if the compression level is invalid (i.e. outside the range [0, 12]).
+ *
+ * Note: for compression, the sliding window size is defined at compilation time
+ * to 32768, the largest size permissible in the DEFLATE format. It cannot be
+ * changed at runtime.
+ *
+ * A single compressor is not safe to use by multiple threads concurrently.
+ * However, different threads may use different compressors concurrently.
+ */
+LIBDEFLATEEXPORT struct libdeflate_compressor * LIBDEFLATEAPI
+libdeflate_alloc_compressor(int compression_level);
+
+/*
+ * libdeflate_deflate_compress() performs raw DEFLATE compression on a buffer of
+ * data. The function attempts to compress 'in_nbytes' bytes of data located at
+ * 'in' and write the results to 'out', which has space for 'out_nbytes_avail'
+ * bytes. The return value is the compressed size in bytes, or 0 if the data
+ * could not be compressed to 'out_nbytes_avail' bytes or fewer.
+ */
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
+ const void *in, size_t in_nbytes,
+ void *out, size_t out_nbytes_avail);
+
+/*
+ * libdeflate_deflate_compress_bound() returns a worst-case upper bound on the
+ * number of bytes of compressed data that may be produced by compressing any
+ * buffer of length less than or equal to 'in_nbytes' using
+ * libdeflate_deflate_compress() with the specified compressor. Mathematically,
+ * this bound will necessarily be a number greater than or equal to 'in_nbytes'.
+ * It may be an overestimate of the true upper bound. The return value is
+ * guaranteed to be the same for all invocations with the same compressor and
+ * same 'in_nbytes'.
+ *
+ * As a special case, 'compressor' may be NULL. This causes the bound to be
+ * taken across *any* libdeflate_compressor that could ever be allocated with
+ * this build of the library, with any options.
+ *
+ * Note that this function is not necessary in many applications. With
+ * block-based compression, it is usually preferable to separately store the
+ * uncompressed size of each block and to store any blocks that did not compress
+ * to less than their original size uncompressed. In that scenario, there is no
+ * need to know the worst-case compressed size, since the maximum number of
+ * bytes of compressed data that may be used would always be one less than the
+ * input length. You can just pass a buffer of that size to
+ * libdeflate_deflate_compress() and store the data uncompressed if
+ * libdeflate_deflate_compress() returns 0, indicating that the compressed data
+ * did not fit into the provided output buffer.
+ */
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
+ size_t in_nbytes);
+
+/*
+ * Like libdeflate_deflate_compress(), but stores the data in the zlib wrapper
+ * format.
+ */
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
+ const void *in, size_t in_nbytes,
+ void *out, size_t out_nbytes_avail);
+
+/*
+ * Like libdeflate_deflate_compress_bound(), but assumes the data will be
+ * compressed with libdeflate_zlib_compress() rather than with
+ * libdeflate_deflate_compress().
+ */
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
+ size_t in_nbytes);
+
+/*
+ * Like libdeflate_deflate_compress(), but stores the data in the gzip wrapper
+ * format.
+ */
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
+ const void *in, size_t in_nbytes,
+ void *out, size_t out_nbytes_avail);
+
+/*
+ * Like libdeflate_deflate_compress_bound(), but assumes the data will be
+ * compressed with libdeflate_gzip_compress() rather than with
+ * libdeflate_deflate_compress().
+ */
+LIBDEFLATEEXPORT size_t LIBDEFLATEAPI
+libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
+ size_t in_nbytes);
+
+/*
+ * libdeflate_free_compressor() frees a compressor that was allocated with
+ * libdeflate_alloc_compressor(). If a NULL pointer is passed in, no action is
+ * taken.
+ */
+LIBDEFLATEEXPORT void LIBDEFLATEAPI
+libdeflate_free_compressor(struct libdeflate_compressor *compressor);
+
+/* ========================================================================== */
+/* Decompression */
+/* ========================================================================== */
+
+struct libdeflate_decompressor;
+
+/*
+ * libdeflate_alloc_decompressor() allocates a new decompressor that can be used
+ * for DEFLATE, zlib, and gzip decompression. The return value is a pointer to
+ * the new decompressor, or NULL if out of memory.
+ *
+ * This function takes no parameters, and the returned decompressor is valid for
+ * decompressing data that was compressed at any compression level and with any
+ * sliding window size.
+ *
+ * A single decompressor is not safe to use by multiple threads concurrently.
+ * However, different threads may use different decompressors concurrently.
+ */
+LIBDEFLATEEXPORT struct libdeflate_decompressor * LIBDEFLATEAPI
+libdeflate_alloc_decompressor(void);
+
+/*
+ * Result of a call to libdeflate_deflate_decompress(),
+ * libdeflate_zlib_decompress(), or libdeflate_gzip_decompress().
+ */
+enum libdeflate_result {
+ /* Decompression was successful. */
+ LIBDEFLATE_SUCCESS = 0,
+
+ /* Decompressed failed because the compressed data was invalid, corrupt,
+ * or otherwise unsupported. */
+ LIBDEFLATE_BAD_DATA = 1,
+
+ /* A NULL 'actual_out_nbytes_ret' was provided, but the data would have
+ * decompressed to fewer than 'out_nbytes_avail' bytes. */
+ LIBDEFLATE_SHORT_OUTPUT = 2,
+
+ /* The data would have decompressed to more than 'out_nbytes_avail'
+ * bytes. */
+ LIBDEFLATE_INSUFFICIENT_SPACE = 3,
+};
+
+/*
+ * libdeflate_deflate_decompress() decompresses the DEFLATE-compressed stream
+ * from the buffer 'in' with compressed size up to 'in_nbytes' bytes. The
+ * uncompressed data is written to 'out', a buffer with size 'out_nbytes_avail'
+ * bytes. If decompression succeeds, then 0 (LIBDEFLATE_SUCCESS) is returned.
+ * Otherwise, a nonzero result code such as LIBDEFLATE_BAD_DATA is returned. If
+ * a nonzero result code is returned, then the contents of the output buffer are
+ * undefined.
+ *
+ * Decompression stops at the end of the DEFLATE stream (as indicated by the
+ * BFINAL flag), even if it is actually shorter than 'in_nbytes' bytes.
+ *
+ * libdeflate_deflate_decompress() can be used in cases where the actual
+ * uncompressed size is known (recommended) or unknown (not recommended):
+ *
+ * - If the actual uncompressed size is known, then pass the actual
+ * uncompressed size as 'out_nbytes_avail' and pass NULL for
+ * 'actual_out_nbytes_ret'. This makes libdeflate_deflate_decompress() fail
+ * with LIBDEFLATE_SHORT_OUTPUT if the data decompressed to fewer than the
+ * specified number of bytes.
+ *
+ * - If the actual uncompressed size is unknown, then provide a non-NULL
+ * 'actual_out_nbytes_ret' and provide a buffer with some size
+ * 'out_nbytes_avail' that you think is large enough to hold all the
+ * uncompressed data. In this case, if the data decompresses to less than
+ * or equal to 'out_nbytes_avail' bytes, then
+ * libdeflate_deflate_decompress() will write the actual uncompressed size
+ * to *actual_out_nbytes_ret and return 0 (LIBDEFLATE_SUCCESS). Otherwise,
+ * it will return LIBDEFLATE_INSUFFICIENT_SPACE if the provided buffer was
+ * not large enough but no other problems were encountered, or another
+ * nonzero result code if decompression failed for another reason.
+ */
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
+ const void *in, size_t in_nbytes,
+ void *out, size_t out_nbytes_avail,
+ size_t *actual_out_nbytes_ret);
+
+/*
+ * Like libdeflate_deflate_decompress(), but adds the 'actual_in_nbytes_ret'
+ * argument. If decompression succeeds and 'actual_in_nbytes_ret' is not NULL,
+ * then the actual compressed size of the DEFLATE stream (aligned to the next
+ * byte boundary) is written to *actual_in_nbytes_ret.
+ */
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
+ const void *in, size_t in_nbytes,
+ void *out, size_t out_nbytes_avail,
+ size_t *actual_in_nbytes_ret,
+ size_t *actual_out_nbytes_ret);
+
+/*
+ * Like libdeflate_deflate_decompress(), but assumes the zlib wrapper format
+ * instead of raw DEFLATE.
+ *
+ * Decompression will stop at the end of the zlib stream, even if it is shorter
+ * than 'in_nbytes'. If you need to know exactly where the zlib stream ended,
+ * use libdeflate_zlib_decompress_ex().
+ */
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
+ const void *in, size_t in_nbytes,
+ void *out, size_t out_nbytes_avail,
+ size_t *actual_out_nbytes_ret);
+
+/*
+ * Like libdeflate_zlib_decompress(), but adds the 'actual_in_nbytes_ret'
+ * argument. If 'actual_in_nbytes_ret' is not NULL and the decompression
+ * succeeds (indicating that the first zlib-compressed stream in the input
+ * buffer was decompressed), then the actual number of input bytes consumed is
+ * written to *actual_in_nbytes_ret.
+ */
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
+ const void *in, size_t in_nbytes,
+ void *out, size_t out_nbytes_avail,
+ size_t *actual_in_nbytes_ret,
+ size_t *actual_out_nbytes_ret);
+
+/*
+ * Like libdeflate_deflate_decompress(), but assumes the gzip wrapper format
+ * instead of raw DEFLATE.
+ *
+ * If multiple gzip-compressed members are concatenated, then only the first
+ * will be decompressed. Use libdeflate_gzip_decompress_ex() if you need
+ * multi-member support.
+ */
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
+ const void *in, size_t in_nbytes,
+ void *out, size_t out_nbytes_avail,
+ size_t *actual_out_nbytes_ret);
+
+/*
+ * Like libdeflate_gzip_decompress(), but adds the 'actual_in_nbytes_ret'
+ * argument. If 'actual_in_nbytes_ret' is not NULL and the decompression
+ * succeeds (indicating that the first gzip-compressed member in the input
+ * buffer was decompressed), then the actual number of input bytes consumed is
+ * written to *actual_in_nbytes_ret.
+ */
+LIBDEFLATEEXPORT enum libdeflate_result LIBDEFLATEAPI
+libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
+ const void *in, size_t in_nbytes,
+ void *out, size_t out_nbytes_avail,
+ size_t *actual_in_nbytes_ret,
+ size_t *actual_out_nbytes_ret);
+
+/*
+ * libdeflate_free_decompressor() frees a decompressor that was allocated with
+ * libdeflate_alloc_decompressor(). If a NULL pointer is passed in, no action
+ * is taken.
+ */
+LIBDEFLATEEXPORT void LIBDEFLATEAPI
+libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
+
+/* ========================================================================== */
+/* Checksums */
+/* ========================================================================== */
+
+/*
+ * libdeflate_adler32() updates a running Adler-32 checksum with 'len' bytes of
+ * data and returns the updated checksum. When starting a new checksum, the
+ * required initial value for 'adler' is 1. This value is also returned when
+ * 'buffer' is specified as NULL.
+ */
+LIBDEFLATEEXPORT uint32_t LIBDEFLATEAPI
+libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
+
+
+/*
+ * libdeflate_crc32() updates a running CRC-32 checksum with 'len' bytes of data
+ * and returns the updated checksum. When starting a new checksum, the required
+ * initial value for 'crc' is 0. This value is also returned when 'buffer' is
+ * specified as NULL.
+ */
+LIBDEFLATEEXPORT uint32_t LIBDEFLATEAPI
+libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
+
+/* ========================================================================== */
+/* Custom memory allocator */
+/* ========================================================================== */
+
+/*
+ * Install a custom memory allocator which libdeflate will use for all memory
+ * allocations. 'malloc_func' is a function that must behave like malloc(), and
+ * 'free_func' is a function that must behave like free().
+ *
+ * There must not be any libdeflate_compressor or libdeflate_decompressor
+ * structures in existence when calling this function.
+ */
+LIBDEFLATEEXPORT void LIBDEFLATEAPI
+libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
+ void (*free_func)(void *));
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* LIBDEFLATE_H */
diff --git a/libretro/Makefile.common b/libretro/Makefile.common
index 7dab17fab4..8d58d198cd 100644
--- a/libretro/Makefile.common
+++ b/libretro/Makefile.common
@@ -398,6 +398,8 @@ SOURCES_CXX += \
SOURCES_C += $(EXTDIR)/sfmt19937/SFMT.c
SOURCES_C += $(EXTDIR)/xxhash.c
+SOURCES_C += $(EXTDIR)/libdeflate/lib/deflate_compress.c \
+ $(EXTDIR)/libdeflate/lib/utils.c
SOURCES_CXX += \
$(EXTDIR)/snappy/snappy-c.cpp \
@@ -743,6 +745,7 @@ SOURCES_CXX += \
$(COREDIR)/ELF/ParamSFO.cpp \
$(COREDIR)/FileSystems/tlzrc.cpp \
$(COREDIR)/FileSystems/BlockDevices.cpp \
+ $(COREDIR)/FileSystems/CSOCompress.cpp \
$(COREDIR)/FileSystems/BlobFileSystem.cpp \
$(COREDIR)/FileSystems/DirectoryFileSystem.cpp \
$(COREDIR)/FileSystems/FileSystem.cpp \