#pragma region CPL License /* Nuclex Native Framework Copyright (C) 2002-2021 Nuclex Development Labs This library is free software; you can redistribute it and/or modify it under the terms of the IBM Common Public License as published by the IBM Corporation; either version 1.0 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the IBM Common Public License for more details. You should have received a copy of the IBM Common Public License along with this library */ #pragma endregion // CPL License // If the library is compiled as a DLL, this ensures symbols are exported #define NUCLEX_STORAGE_SOURCE 1 #include "ZPaqCompressor.h" #if defined(NUCLEX_STORAGE_HAVE_ZPAQ) #include "Nuclex/Storage/Binary/InputStream.h" #include "Nuclex/Storage/Binary/OutputStream.h" #include #include #include // ZPaq method is: // l, level (0-5 compression level) // factor, block size 0-11 (influences 'type') // , redundancy 0..255 (?) // summand, type (0 = binary) (influences 'type') // // type is 512 if not specified // type is second argument x 4 + third argument if second argument is specified // // const unsigned n=in->size(); // input size // const int arg0=MAX(lg(n+4095)-20, 0); // block size // // Different levels expand another (internal) "method" string depending on input // stream size. Perhaps we should force-buffer up to 1024 / 4096 bytes before // starting compression, then pick parameters according to input size? // namespace libzpaq { // ------------------------------------------------------------------------------------------- // /// Creates a compressor configuration for ZPaq /// Compression level, method and window sizes /// Additional args parsed from the the compression level /// /// This is an internal ZPaq method called by compressBlock() to form a config. /// It seems to be vital to achieve the extreme compression levels ZPaq is capable /// of, but the only way to use it through "official&qout; channels is to /// call compressBlock(), which requires all data to be available upfront or going /// overboard running the compressor in a thread and blocking until it is fed data. /// std::string makeConfig(const char *method, int args[]); // ------------------------------------------------------------------------------------------- // } namespace Nuclex { namespace Storage { namespace Compression { namespace ZPaq { // ------------------------------------------------------------------------------------------- // ZPaqCompressor::ZPaqCompressor(int level) : compressor(), isFinishing(false) { #if defined(USE_LONG_ZPAQ_COMPRESSION_SETUP) // Method strings built by compressBlock() static const std::string methods[] = { "00,0", "x0,1,5,0,3,20", "x0,1,4,0,7,21,1", "x0,2,12,0,7,21,1c0,0,511i2", "x0,0ci1,1,1,1,2am", "x0,0w1i1c256ci1,1,1,1,1,1,2ac0,2,0,255i1c0,3,0,0,255i1c0,4,0,0,0,255i1mm16ts19t0" }; #endif this->compressor.setInput(&this->reader); this->compressor.setOutput(&this->writer); #if defined(USE_LONG_ZPAQ_COMPRESSION_SETUP) level = std::min(std::max(level, 0), 5); int args[9]={0}; std::string config = libzpaq::makeConfig(methods[level].c_str(), args); this->compressor.startBlock(config.c_str(), args, &this->compressionCommandBuffer); #else level = std::min(std::max(level / 2 + 1, 0), 3); this->compressor.startBlock(level); #endif this->compressor.startSegment(); } // ------------------------------------------------------------------------------------------- // ZPaqCompressor::~ZPaqCompressor() { } // ------------------------------------------------------------------------------------------- // StopReason ZPaqCompressor::Process( const std::uint8_t *uncompressedBuffer, std::size_t &uncompressedByteCount, std::uint8_t *outputBuffer, std::size_t &outputByteCount ) { // Because we cannot stop the ZPaq compressor until it has processed all input // bytes, it may generate more output than we want. This will have been saved by // our special buffer writer. If there's still output waiting in the buffer writer, // it'll be written to the output buffer here std::size_t overflowedByteCount = this->writer.TargetNewOutputBuffer( outputBuffer, outputByteCount ); if(overflowedByteCount >= outputByteCount) { return StopReason::OutputBufferFull; // outputByteCount can remain as-is } // Select the new input buffer and let libzpaq do the compression this->reader.SetInputBuffer(uncompressedBuffer, uncompressedByteCount); assert(uncompressedByteCount < std::numeric_limits::max()); bool moreInputFollowing = this->compressor.compress(static_cast(uncompressedByteCount)); if(!moreInputFollowing) { throw std::logic_error(u8"libzpaq accessed more bytes than told to"); } std::size_t remainingOutputByteCount = this->writer.GetRemainingByteCount(); if(remainingOutputByteCount == 0) { return StopReason::OutputBufferFull; } else { outputByteCount -= remainingOutputByteCount; return StopReason::InputBufferExhausted; } } // ------------------------------------------------------------------------------------------- // StopReason ZPaqCompressor::Finish( std::uint8_t *outputBuffer, std::size_t &outputByteCount ) { // Because we cannot stop the ZPaq compressor until it has processed all input // bytes, it may generate more output than we want. This will have been saved by // our special buffer writer. If there's still output waiting in the buffer writer, // it'll be written to the output buffer here std::size_t overflowedByteCount = this->writer.TargetNewOutputBuffer( outputBuffer, outputByteCount ); if(overflowedByteCount >= outputByteCount) { return StopReason::OutputBufferFull; // outputByteCount can remain as-is } if(!this->isFinishing) { this->compressor.endSegment(); this->compressor.endBlock(); this->isFinishing = true; } if(this->writer.HasOverflowBytes()) { return StopReason::OutputBufferFull; } else { outputByteCount -= this->writer.GetRemainingByteCount(); return StopReason::Finished; } } // ------------------------------------------------------------------------------------------- // }}}} // namespace Nuclex::Storage::Compression::LZip #endif // defined(NUCLEX_STORAGE_HAVE_ZPAQ)