diff --git a/C/7zVersion.h b/C/7zVersion.h index ddcc62b7..46e3aca0 100644 --- a/C/7zVersion.h +++ b/C/7zVersion.h @@ -1,7 +1,7 @@ #define MY_VER_MAJOR 17 #define MY_VER_MINOR 00 #define MY_VER_BUILD 0 -#define MY_VERSION_NUMBERS "17.00 ZS v1.2.0 R4" +#define MY_VERSION_NUMBERS "17.00 ZS v1.2.0 R5" #define MY_VERSION MY_VERSION_NUMBERS #ifdef MY_CPU_NAME @@ -10,7 +10,7 @@ #define MY_VERSION_CPU MY_VERSION #endif -#define MY_DATE "2017-05-27" +#define MY_DATE "2017-05-28" #undef MY_COPYRIGHT #undef MY_VERSION_COPYRIGHT_DATE #define MY_AUTHOR_NAME "Igor Pavlov, Tino Reichardt" diff --git a/C/7zVersionTr.h b/C/7zVersionTr.h index f1898f4e..f1367842 100644 --- a/C/7zVersionTr.h +++ b/C/7zVersionTr.h @@ -1,9 +1,9 @@ #define MY_VER_MAJOR 1 #define MY_VER_MINOR 2 #define MY_VER_BUILD 0 -#define MY_VERSION_NUMBERS "1.2.0 R4" +#define MY_VERSION_NUMBERS "1.2.0 R5" #define MY_VERSION MY_VERSION_NUMBERS -#define MY_DATE "2017-05-27" +#define MY_DATE "2017-05-28" #undef MY_COPYRIGHT #undef MY_VERSION_COPYRIGHT_DATE #define MY_AUTHOR_NAME "Tino Reichardt" diff --git a/C/lizard/LICENSE b/C/lizard/LICENSE index 048135e5..b335b0ba 100644 --- a/C/lizard/LICENSE +++ b/C/lizard/LICENSE @@ -1,6 +1,6 @@ -LZ5 Library +Lizard Library Copyright (C) 2011-2016, Yann Collet. -Copyright (C) 2015-2016, Przemyslaw Skibinski +Copyright (C) 2016-2017, Przemyslaw Skibinski All rights reserved. Redistribution and use in source and binary forms, with or without modification, diff --git a/C/lizard/LZ5_to_LIZ.sh b/C/lizard/LZ5_to_LIZ.sh deleted file mode 100644 index 936e499c..00000000 --- a/C/lizard/LZ5_to_LIZ.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh -# C/lizard/* -# /TR 2017-05-25 - -for i in *.c *.h; do - sed -i 's/LZ5_/LIZ_/g' "$i" - sed -i 's/LZ5F_/LIZF_/g' "$i" - sed -i 's/"lz5_/"liz_/g' "$i" -done - -for f in lz5*; do - l=`echo $f|sed -e 's/lz5/liz/g'` - mv $f $l -done diff --git a/C/lizard/README.md b/C/lizard/README.md index 8bad6357..2775b984 100644 --- a/C/lizard/README.md +++ b/C/lizard/README.md @@ -1,95 +1,76 @@ -LZ5 - efficient compression with very fast decompression --------------------------------------------------------- +Lizard - Library Files +================================ -LZ5 is a lossless compression algorithm which contains 4 compression methods: -- fastLZ4 : compression levels -10...-19 are designed to give better decompression speed than [LZ4] i.e. over 2000 MB/s -- LZ5v2 : compression levels -20...-29 are designed to give better ratio than [LZ4] keeping 75% decompression speed -- fastLZ4 + Huffman : compression levels -30...-39 add Huffman coding to fastLZ4 -- LZ5v2 + Huffman : compression levels -40...-49 give the best ratio (comparable to [zlib] and low levels of [zstd]/[brotli]) at decompression speed of 1000 MB/s - -LZ5 library is based on frequently used [LZ4] library by Yann Collet. -LZ5 library is provided as open-source software using BSD 2-Clause license. -The high compression/decompression speed is achieved without any SSE and AVX extensions. +The __lib__ directory contains several directories. +Depending on target use case, it's enough to include only files from relevant directories. -|Branch |Status | -|------------|---------| -|lz5_v1.5 | [![Build Status][travis15Badge]][travisLink] [![Build status][Appveyor15Badge]][AppveyorLink] | -|lz5_v2.0 | [![Build Status][travis20Badge]][travisLink] [![Build status][Appveyor20Badge]][AppveyorLink] | +#### API -[travis15Badge]: https://travis-ci.org/inikep/lz5.svg?branch=lz5_v1.5 "Continuous Integration test suite" -[travis20Badge]: https://travis-ci.org/inikep/lz5.svg?branch=lz5_v2.0 "Continuous Integration test suite" -[travisLink]: https://travis-ci.org/inikep/lz5 -[Appveyor15Badge]: https://ci.appveyor.com/api/projects/status/o0ib75nwokjiui36/branch/lz5_v1.5?svg=true "Visual test suite" -[Appveyor20Badge]: https://ci.appveyor.com/api/projects/status/o0ib75nwokjiui36/branch/lz5_v2.0?svg=true "Visual test suite" -[AppveyorLink]: https://ci.appveyor.com/project/inikep/lz5 -[LZ4]: https://github.com/lz4/lz4 -[zlib]: https://github.com/madler/zlib -[zstd]: https://github.com/facebook/zstd -[brotli]: https://github.com/google/brotli +Lizard stable API is exposed within [lizard_compress.h](lizard_compress.h) and [lizard_decompress.h](lizard_decompress.h), +at the root of `lib` directory. -Benchmarks -------------------------- - -The following results are obtained with [lzbench](https://github.com/inikep/lzbench) and `-t16,16` -using 1 core of Intel Core i5-4300U, Windows 10 64-bit (MinGW-w64 compilation under gcc 6.2.0) -with [silesia.tar] which contains tarred files from [Silesia compression corpus](http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia). - -| Compressor name | Compression| Decompress.| Compr. size | Ratio | -| --------------- | -----------| -----------| ----------- | ----- | -| memcpy | 7332 MB/s | 8719 MB/s | 211947520 |100.00 | -| lz4 1.7.3 | 440 MB/s | 2318 MB/s | 100880800 | 47.60 | -| lz4hc 1.7.3 -1 | 98 MB/s | 2121 MB/s | 87591763 | 41.33 | -| lz4hc 1.7.3 -4 | 55 MB/s | 2259 MB/s | 79807909 | 37.65 | -| lz4hc 1.7.3 -9 | 22 MB/s | 2315 MB/s | 77892285 | 36.75 | -| lz4hc 1.7.3 -12 | 17 MB/s | 2323 MB/s | 77849762 | 36.73 | -| lz4hc 1.7.3 -16 | 10 MB/s | 2323 MB/s | 77841782 | 36.73 | -| lz5 2.0 -10 | 346 MB/s | 2610 MB/s | 103402971 | 48.79 | -| lz5 2.0 -12 | 103 MB/s | 2458 MB/s | 86232422 | 40.69 | -| lz5 2.0 -15 | 50 MB/s | 2552 MB/s | 81187330 | 38.31 | -| lz5 2.0 -19 | 3.04 MB/s | 2497 MB/s | 77416400 | 36.53 | -| lz5 2.0 -21 | 157 MB/s | 1795 MB/s | 89239174 | 42.10 | -| lz5 2.0 -23 | 30 MB/s | 1778 MB/s | 81097176 | 38.26 | -| lz5 2.0 -26 | 6.63 MB/s | 1734 MB/s | 74503695 | 35.15 | -| lz5 2.0 -29 | 1.37 MB/s | 1634 MB/s | 68694227 | 32.41 | -| lz5 2.0 -30 | 246 MB/s | 909 MB/s | 85727429 | 40.45 | -| lz5 2.0 -32 | 94 MB/s | 1244 MB/s | 76929454 | 36.30 | -| lz5 2.0 -35 | 47 MB/s | 1435 MB/s | 73850400 | 34.84 | -| lz5 2.0 -39 | 2.94 MB/s | 1502 MB/s | 69807522 | 32.94 | -| lz5 2.0 -41 | 126 MB/s | 961 MB/s | 76100661 | 35.91 | -| lz5 2.0 -43 | 28 MB/s | 1101 MB/s | 70955653 | 33.48 | -| lz5 2.0 -46 | 6.25 MB/s | 1073 MB/s | 65413061 | 30.86 | -| lz5 2.0 -49 | 1.27 MB/s | 1064 MB/s | 60679215 | 28.63 | -| zlib 1.2.8 -1 | 66 MB/s | 244 MB/s | 77259029 | 36.45 | -| zlib 1.2.8 -6 | 20 MB/s | 263 MB/s | 68228431 | 32.19 | -| zlib 1.2.8 -9 | 8.37 MB/s | 266 MB/s | 67644548 | 31.92 | -| zstd 1.1.1 -1 | 235 MB/s | 645 MB/s | 73659468 | 34.75 | -| zstd 1.1.1 -2 | 181 MB/s | 600 MB/s | 70168955 | 33.11 | -| zstd 1.1.1 -5 | 88 MB/s | 565 MB/s | 65002208 | 30.67 | -| zstd 1.1.1 -8 | 31 MB/s | 619 MB/s | 61026497 | 28.79 | -| zstd 1.1.1 -11 | 16 MB/s | 613 MB/s | 59523167 | 28.08 | -| zstd 1.1.1 -15 | 4.97 MB/s | 639 MB/s | 58007773 | 27.37 | -| zstd 1.1.1 -18 | 2.87 MB/s | 583 MB/s | 55294241 | 26.09 | -| zstd 1.1.1 -22 | 1.44 MB/s | 505 MB/s | 52731930 | 24.88 | -| brotli 0.5.2 -0 | 217 MB/s | 244 MB/s | 78226979 | 36.91 | -| brotli 0.5.2 -2 | 96 MB/s | 283 MB/s | 68066621 | 32.11 | -| brotli 0.5.2 -5 | 24 MB/s | 312 MB/s | 60801716 | 28.69 | -| brotli 0.5.2 -8 | 5.56 MB/s | 324 MB/s | 57382470 | 27.07 | -| brotli 0.5.2 -11 | 0.39 MB/s | 266 MB/s | 51138054 | 24.13 | - -[silesia.tar]: https://drive.google.com/file/d/0BwX7dtyRLxThenZpYU9zLTZhR1k/view?usp=sharing - - -Documentation -------------------------- - -The raw LZ5 block compression format is detailed within [lz5_Block_format]. +#### Compatibility issues +The raw Lizard block compression format is detailed within [lizard_Block_format]. To compress an arbitrarily long file or data stream, multiple blocks are required. Organizing these blocks and providing a common header format to handle their content -is the purpose of the Frame format, defined into [lz5_Frame_format]. -Interoperable versions of LZ5 must respect this frame format. +is the purpose of the Frame format, defined in [lizard_Frame_format]. +`lizard` command line utility produces files or streams compatible with the Frame format. +(_Advanced stuff_ : It's possible to hide xxhash symbols into a local namespace. +This is what `liblizard` does, to avoid symbol duplication +in case a user program would link to several libraries containing xxhash symbols.) -[lz5_Block_format]: doc/lz5_Block_format.md -[lz5_Frame_format]: doc/lz5_Frame_format.md +[lizard_Block_format]: ../doc/lizard_Block_format.md +[lizard_Frame_format]: ../doc/lizard_Frame_format.md + + +#### Various Lizard builds + +Files `lizard_common.h`, `lizard_compress*`, `lizard_parser_*.h`, `lizard_decompress*`, and `entropy\mem.h` are required in all circumstances. + +To compile: +- Lizard_raw only with levels 10...29 : use the `-DLIZARD_NO_HUFFMAN` compiler flag +- Lizard_raw with levels 10...49 : include also all files from `entropy` directory +- Lizard_frame with levels 10...49 : `lizard_frame*` and all files from `entropy` and `xxhash` directories + + +#### Advanced API + +A more complex `lizard_frame_static.h` is also provided. +It contains definitions which are not guaranteed to remain stable within future versions. +It must be used with static linking ***only***. + + +#### Using MinGW+MSYS to create DLL + +DLL can be created using MinGW+MSYS with the `make liblizard` command. +This command creates `dll\liblizard.dll` and the import library `dll\liblizard.lib`. +The import library is only required with Visual C++. +The header files `lizard.h`, `lizardhc.h`, `lizard_frame.h` and the dynamic library +`dll\liblizard.dll` are required to compile a project using gcc/MinGW. +The dynamic library has to be added to linking options. +It means that if a project that uses Lizard consists of a single `test-dll.c` +file it should be compiled with `liblizard.lib`. For example: +``` + gcc $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\liblizard.dll +``` +The compiled executable will require Lizard DLL which is available at `dll\liblizard.dll`. + + +#### Miscellaneous + +Other files present in the directory are not source code. There are : + + - LICENSE : contains the BSD license text + - Makefile : script to compile or install lizard library (static or dynamic) + - liblizard.pc.in : for pkg-config (make install) + - README.md : this file + + +#### License + +All source material within __lib__ directory are BSD 2-Clause licensed. +See [LICENSE](LICENSE) for details. +The license is also repeated at the top of each source file. diff --git a/C/lizard/bitstream.h b/C/lizard/bitstream.h deleted file mode 100644 index e96798fe..00000000 --- a/C/lizard/bitstream.h +++ /dev/null @@ -1,414 +0,0 @@ -/* ****************************************************************** - bitstream - Part of FSE library - header file (to include) - Copyright (C) 2013-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Source repository : https://github.com/Cyan4973/FiniteStateEntropy -****************************************************************** */ -#ifndef BITSTREAM_H_MODULE -#define BITSTREAM_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - - -/* -* This API consists of small unitary functions, which must be inlined for best performance. -* Since link-time-optimization is not available for all compilers, -* these functions are defined into a .h to be included. -*/ - -/*-**************************************** -* Dependencies -******************************************/ -#include "mem.h" /* unaligned access routines */ -#include "error_private.h" /* error codes and messages */ - - -/*========================================= -* Target specific -=========================================*/ -#if defined(__BMI__) && defined(__GNUC__) -# include /* support for bextr (experimental) */ -#endif - - -/*-****************************************** -* bitStream encoding API (write forward) -********************************************/ -/* bitStream can mix input from multiple sources. -* A critical property of these streams is that they encode and decode in **reverse** direction. -* So the first bit sequence you add will be the last to be read, like a LIFO stack. -*/ -typedef struct -{ - size_t bitContainer; - int bitPos; - char* startPtr; - char* ptr; - char* endPtr; -} BIT_CStream_t; - -MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity); -MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits); -MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC); -MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); - -/* Start with initCStream, providing the size of buffer to write into. -* bitStream will never write outside of this buffer. -* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code. -* -* bits are first added to a local register. -* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. -* Writing data into memory is an explicit operation, performed by the flushBits function. -* Hence keep track how many bits are potentially stored into local register to avoid register overflow. -* After a flushBits, a maximum of 7 bits might still be stored into local register. -* -* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers. -* -* Last operation is to close the bitStream. -* The function returns the final size of CStream in bytes. -* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable) -*/ - - -/*-******************************************** -* bitStream decoding API (read backward) -**********************************************/ -typedef struct -{ - size_t bitContainer; - unsigned bitsConsumed; - const char* ptr; - const char* start; -} BIT_DStream_t; - -typedef enum { BIT_DStream_unfinished = 0, - BIT_DStream_endOfBuffer = 1, - BIT_DStream_completed = 2, - BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */ - /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ - -MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize); -MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); -MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD); -MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); - - -/* Start by invoking BIT_initDStream(). -* A chunk of the bitStream is then stored into a local register. -* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). -* You can then retrieve bitFields stored into the local register, **in reverse order**. -* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. -* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. -* Otherwise, it can be less than that, so proceed accordingly. -* Checking if DStream has reached its end can be performed with BIT_endOfDStream(). -*/ - - -/*-**************************************** -* unsafe API -******************************************/ -MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits); -/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */ - -MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); -/* unsafe version; does not check buffer overflow */ - -MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); -/* faster, but works only if nbBits >= 1 */ - - - -/*-************************************************************** -* Internal functions -****************************************************************/ -MEM_STATIC unsigned BIT_highbit32 (register U32 val) -{ -# if defined(_MSC_VER) /* Visual */ - unsigned long r=0; - _BitScanReverse ( &r, val ); - return (unsigned) r; -# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ - return 31 - __builtin_clz (val); -# else /* Software version */ - static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; - U32 v = val; - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27]; -# endif -} - -/*===== Local Constants =====*/ -static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */ - - -/*-************************************************************** -* bitStream encoding -****************************************************************/ -/*! BIT_initCStream() : - * `dstCapacity` must be > sizeof(void*) - * @return : 0 if success, - otherwise an error code (can be tested using ERR_isError() ) */ -MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* startPtr, size_t dstCapacity) -{ - bitC->bitContainer = 0; - bitC->bitPos = 0; - bitC->startPtr = (char*)startPtr; - bitC->ptr = bitC->startPtr; - bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->ptr); - if (dstCapacity <= sizeof(bitC->ptr)) return ERROR(dstSize_tooSmall); - return 0; -} - -/*! BIT_addBits() : - can add up to 26 bits into `bitC`. - Does not check for register overflow ! */ -MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits) -{ - bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos; - bitC->bitPos += nbBits; -} - -/*! BIT_addBitsFast() : - * works only if `value` is _clean_, meaning all high bits above nbBits are 0 */ -MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits) -{ - bitC->bitContainer |= value << bitC->bitPos; - bitC->bitPos += nbBits; -} - -/*! BIT_flushBitsFast() : - * unsafe version; does not check buffer overflow */ -MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC) -{ - size_t const nbBytes = bitC->bitPos >> 3; - MEM_writeLEST(bitC->ptr, bitC->bitContainer); - bitC->ptr += nbBytes; - bitC->bitPos &= 7; - bitC->bitContainer >>= nbBytes*8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */ -} - -/*! BIT_flushBits() : - * safe version; check for buffer overflow, and prevents it. - * note : does not signal buffer overflow. This will be revealed later on using BIT_closeCStream() */ -MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC) -{ - size_t const nbBytes = bitC->bitPos >> 3; - MEM_writeLEST(bitC->ptr, bitC->bitContainer); - bitC->ptr += nbBytes; - if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; - bitC->bitPos &= 7; - bitC->bitContainer >>= nbBytes*8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */ -} - -/*! BIT_closeCStream() : - * @return : size of CStream, in bytes, - or 0 if it could not fit into dstBuffer */ -MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) -{ - BIT_addBitsFast(bitC, 1, 1); /* endMark */ - BIT_flushBits(bitC); - - if (bitC->ptr >= bitC->endPtr) return 0; /* doesn't fit within authorized budget : cancel */ - - return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); -} - - -/*-******************************************************** -* bitStream decoding -**********************************************************/ -/*! BIT_initDStream() : -* Initialize a BIT_DStream_t. -* `bitD` : a pointer to an already allocated BIT_DStream_t structure. -* `srcSize` must be the *exact* size of the bitStream, in bytes. -* @return : size of stream (== srcSize) or an errorCode if a problem is detected -*/ -MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize) -{ - if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } - - if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ - bitD->start = (const char*)srcBuffer; - bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); - bitD->bitContainer = MEM_readLEST(bitD->ptr); - { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; - bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; - if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } - } else { - bitD->start = (const char*)srcBuffer; - bitD->ptr = bitD->start; - bitD->bitContainer = *(const BYTE*)(bitD->start); - switch(srcSize) - { - case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); - case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); - case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); - case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; - case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; - case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; - default:; - } - { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; - bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; - if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } - bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; - } - - return srcSize; -} - -MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start) -{ - return bitContainer >> start; -} - -MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) -{ -#if defined(__BMI__) && defined(__GNUC__) /* experimental */ -# if defined(__x86_64__) - if (sizeof(bitContainer)==8) - return _bextr_u64(bitContainer, start, nbBits); - else -# endif - return _bextr_u32(bitContainer, start, nbBits); -#else - return (bitContainer >> start) & BIT_mask[nbBits]; -#endif -} - -MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) -{ - return bitContainer & BIT_mask[nbBits]; -} - -/*! BIT_lookBits() : - * Provides next n bits from local register. - * local register is not modified. - * On 32-bits, maxNbBits==24. - * On 64-bits, maxNbBits==56. - * @return : value extracted - */ - MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) -{ -#if defined(__BMI__) && defined(__GNUC__) /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */ - return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits); -#else - U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; - return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask); -#endif -} - -/*! BIT_lookBitsFast() : -* unsafe version; only works only if nbBits >= 1 */ -MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) -{ - U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; - return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask); -} - -MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) -{ - bitD->bitsConsumed += nbBits; -} - -/*! BIT_readBits() : - * Read (consume) next n bits from local register and update. - * Pay attention to not read more than nbBits contained into local register. - * @return : extracted value. - */ -MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits) -{ - size_t const value = BIT_lookBits(bitD, nbBits); - BIT_skipBits(bitD, nbBits); - return value; -} - -/*! BIT_readBitsFast() : -* unsafe version; only works only if nbBits >= 1 */ -MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits) -{ - size_t const value = BIT_lookBitsFast(bitD, nbBits); - BIT_skipBits(bitD, nbBits); - return value; -} - -/*! BIT_reloadDStream() : -* Refill `BIT_DStream_t` from src buffer previously defined (see BIT_initDStream() ). -* This function is safe, it guarantees it will not read beyond src buffer. -* @return : status of `BIT_DStream_t` internal register. - if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */ -MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) -{ - if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */ - return BIT_DStream_overflow; - - if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) { - bitD->ptr -= bitD->bitsConsumed >> 3; - bitD->bitsConsumed &= 7; - bitD->bitContainer = MEM_readLEST(bitD->ptr); - return BIT_DStream_unfinished; - } - if (bitD->ptr == bitD->start) { - if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; - return BIT_DStream_completed; - } - { U32 nbBytes = bitD->bitsConsumed >> 3; - BIT_DStream_status result = BIT_DStream_unfinished; - if (bitD->ptr - nbBytes < bitD->start) { - nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ - result = BIT_DStream_endOfBuffer; - } - bitD->ptr -= nbBytes; - bitD->bitsConsumed -= nbBytes*8; - bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */ - return result; - } -} - -/*! BIT_endOfDStream() : -* @return Tells if DStream has exactly reached its end (all bits consumed). -*/ -MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) -{ - return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8)); -} - -#if defined (__cplusplus) -} -#endif - -#endif /* BITSTREAM_H_MODULE */ diff --git a/C/lizard/error_private.h b/C/lizard/error_private.h deleted file mode 100644 index 88906149..00000000 --- a/C/lizard/error_private.h +++ /dev/null @@ -1,125 +0,0 @@ -/* ****************************************************************** - Error codes and messages - Copyright (C) 2013-2016, Yann Collet - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Homepage : http://www.zstd.net -****************************************************************** */ -/* Note : this module is expected to remain private, do not expose it */ - -#ifndef ERROR_H_MODULE -#define ERROR_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - - -/* **************************************** -* Dependencies -******************************************/ -#include /* size_t */ -#include "error_public.h" /* enum list */ - - -/* **************************************** -* Compiler-specific -******************************************/ -#if defined(__GNUC__) -# define ERR_STATIC static __attribute__((unused)) -#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define ERR_STATIC static inline -#elif defined(_MSC_VER) -# define ERR_STATIC static __inline -#else -# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ -#endif - - -/*-**************************************** -* Customization (error_public.h) -******************************************/ -typedef ZSTD_ErrorCode ERR_enum; -#define PREFIX(name) ZSTD_error_##name - - -/*-**************************************** -* Error codes handling -******************************************/ -#ifdef ERROR -# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ -#endif -#define ERROR(name) ((size_t)-PREFIX(name)) - -ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } - -ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } - - -/*-**************************************** -* Error Strings -******************************************/ - -ERR_STATIC const char* ERR_getErrorString(ERR_enum code) -{ - static const char* notErrorCode = "Unspecified error code"; - switch( code ) - { - case PREFIX(no_error): return "No error detected"; - case PREFIX(GENERIC): return "Error (generic)"; - case PREFIX(prefix_unknown): return "Unknown frame descriptor"; - case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; - case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; - case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound"; - case PREFIX(init_missing): return "Context should be init first"; - case PREFIX(memory_allocation): return "Allocation error : not enough memory"; - case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; - case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; - case PREFIX(srcSize_wrong): return "Src size incorrect"; - case PREFIX(corruption_detected): return "Corrupted block detected"; - case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; - case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; - case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; - case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; - case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; - case PREFIX(dictionary_wrong): return "Dictionary mismatch"; - case PREFIX(maxCode): - default: return notErrorCode; - } -} - -ERR_STATIC const char* ERR_getErrorName(size_t code) -{ - return ERR_getErrorString(ERR_getErrorCode(code)); -} - -#if defined (__cplusplus) -} -#endif - -#endif /* ERROR_H_MODULE */ diff --git a/C/lizard/error_public.h b/C/lizard/error_public.h deleted file mode 100644 index 29050b3b..00000000 --- a/C/lizard/error_public.h +++ /dev/null @@ -1,77 +0,0 @@ -/* ****************************************************************** - Error codes list - Copyright (C) 2016, Yann Collet - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Homepage : http://www.zstd.net -****************************************************************** */ -#ifndef ERROR_PUBLIC_H_MODULE -#define ERROR_PUBLIC_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - - -/* **************************************** -* error codes list -******************************************/ -typedef enum { - ZSTD_error_no_error, - ZSTD_error_GENERIC, - ZSTD_error_prefix_unknown, - ZSTD_error_frameParameter_unsupported, - ZSTD_error_frameParameter_unsupportedBy32bits, - ZSTD_error_compressionParameter_unsupported, - ZSTD_error_init_missing, - ZSTD_error_memory_allocation, - ZSTD_error_stage_wrong, - ZSTD_error_dstSize_tooSmall, - ZSTD_error_srcSize_wrong, - ZSTD_error_corruption_detected, - ZSTD_error_checksum_wrong, - ZSTD_error_tableLog_tooLarge, - ZSTD_error_maxSymbolValue_tooLarge, - ZSTD_error_maxSymbolValue_tooSmall, - ZSTD_error_dictionary_corrupted, - ZSTD_error_dictionary_wrong, - ZSTD_error_maxCode -} ZSTD_ErrorCode; - -/*! ZSTD_getErrorCode() : - convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, - which can be used to compare directly with enum list published into "error_public.h" */ -ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); -const char* ZSTD_getErrorString(ZSTD_ErrorCode code); - - -#if defined (__cplusplus) -} -#endif - -#endif /* ERROR_PUBLIC_H_MODULE */ diff --git a/C/lizard/fse.h b/C/lizard/fse.h deleted file mode 100644 index 98241563..00000000 --- a/C/lizard/fse.h +++ /dev/null @@ -1,628 +0,0 @@ -/* ****************************************************************** - FSE : Finite State Entropy codec - Public Prototypes declaration - Copyright (C) 2013-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Source repository : https://github.com/Cyan4973/FiniteStateEntropy -****************************************************************** */ -#ifndef LIZFSE_H -#define LIZFSE_H - -#if defined (__cplusplus) -extern "C" { -#endif - - -/*-***************************************** -* Dependencies -******************************************/ -#include /* size_t, ptrdiff_t */ - - -/*-**************************************** -* FSE simple functions -******************************************/ -/*! LIZFSE_compress() : - Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'. - 'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= LIZFSE_compressBound(srcSize). - @return : size of compressed data (<= dstCapacity). - Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! - if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead. - if LIZFSE_isError(return), compression failed (more details using LIZFSE_getErrorName()) -*/ -size_t LIZFSE_compress(void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - -/*! LIZFSE_decompress(): - Decompress FSE data from buffer 'cSrc', of size 'cSrcSize', - into already allocated destination buffer 'dst', of size 'dstCapacity'. - @return : size of regenerated data (<= maxDstSize), - or an error code, which can be tested using LIZFSE_isError() . - - ** Important ** : LIZFSE_decompress() does not decompress non-compressible nor RLE data !!! - Why ? : making this distinction requires a header. - Header management is intentionally delegated to the user layer, which can better manage special cases. -*/ -size_t LIZFSE_decompress(void* dst, size_t dstCapacity, - const void* cSrc, size_t cSrcSize); - - -/*-***************************************** -* Tool functions -******************************************/ -size_t LIZFSE_compressBound(size_t size); /* maximum compressed size */ - -/* Error Management */ -unsigned LIZFSE_isError(size_t code); /* tells if a return value is an error code */ -const char* LIZFSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */ - - -/*-***************************************** -* FSE advanced functions -******************************************/ -/*! LIZFSE_compress2() : - Same as LIZFSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog' - Both parameters can be defined as '0' to mean : use default value - @return : size of compressed data - Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!! - if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression. - if LIZFSE_isError(return), it's an error code. -*/ -size_t LIZFSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); - - -/*-***************************************** -* FSE detailed API -******************************************/ -/*! -LIZFSE_compress() does the following: -1. count symbol occurrence from source[] into table count[] -2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog) -3. save normalized counters to memory buffer using writeNCount() -4. build encoding table 'CTable' from normalized counters -5. encode the data stream using encoding table 'CTable' - -LIZFSE_decompress() does the following: -1. read normalized counters with readNCount() -2. build decoding table 'DTable' from normalized counters -3. decode the data stream using decoding table 'DTable' - -The following API allows targeting specific sub-functions for advanced tasks. -For example, it's possible to compress several blocks using the same 'CTable', -or to save and provide normalized distribution using external method. -*/ - -/* *** COMPRESSION *** */ - -/*! LIZFSE_count(): - Provides the precise count of each byte within a table 'count'. - 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1). - *maxSymbolValuePtr will be updated if detected smaller than initial value. - @return : the count of the most frequent symbol (which is not identified). - if return == srcSize, there is only one symbol. - Can also return an error code, which can be tested with LIZFSE_isError(). */ -size_t LIZFSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); - -/*! LIZFSE_optimalTableLog(): - dynamically downsize 'tableLog' when conditions are met. - It saves CPU time, by using smaller tables, while preserving or even improving compression ratio. - @return : recommended tableLog (necessarily <= 'maxTableLog') */ -unsigned LIZFSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); - -/*! LIZFSE_normalizeCount(): - normalize counts so that sum(count[]) == Power_of_2 (2^tableLog) - 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). - @return : tableLog, - or an errorCode, which can be tested using LIZFSE_isError() */ -size_t LIZFSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue); - -/*! LIZFSE_NCountWriteBound(): - Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'. - Typically useful for allocation purpose. */ -size_t LIZFSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog); - -/*! LIZFSE_writeNCount(): - Compactly save 'normalizedCounter' into 'buffer'. - @return : size of the compressed table, - or an errorCode, which can be tested using LIZFSE_isError(). */ -size_t LIZFSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); - - -/*! Constructor and Destructor of LIZFSE_CTable. - Note that LIZFSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ -typedef unsigned LIZFSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ -LIZFSE_CTable* LIZFSE_createCTable (unsigned tableLog, unsigned maxSymbolValue); -void LIZFSE_freeCTable (LIZFSE_CTable* ct); - -/*! LIZFSE_buildCTable(): - Builds `ct`, which must be already allocated, using LIZFSE_createCTable(). - @return : 0, or an errorCode, which can be tested using LIZFSE_isError() */ -size_t LIZFSE_buildCTable(LIZFSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); - -/*! LIZFSE_compress_usingCTable(): - Compress `src` using `ct` into `dst` which must be already allocated. - @return : size of compressed data (<= `dstCapacity`), - or 0 if compressed data could not fit into `dst`, - or an errorCode, which can be tested using LIZFSE_isError() */ -size_t LIZFSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const LIZFSE_CTable* ct); - -/*! -Tutorial : ----------- -The first step is to count all symbols. LIZFSE_count() does this job very fast. -Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells. -'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0] -maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value) -LIZFSE_count() will return the number of occurrence of the most frequent symbol. -This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility. -If there is an error, the function will return an ErrorCode (which can be tested using LIZFSE_isError()). - -The next step is to normalize the frequencies. -LIZFSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'. -It also guarantees a minimum of 1 to any Symbol with frequency >= 1. -You can use 'tableLog'==0 to mean "use default tableLog value". -If you are unsure of which tableLog value to use, you can ask LIZFSE_optimalTableLog(), -which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default"). - -The result of LIZFSE_normalizeCount() will be saved into a table, -called 'normalizedCounter', which is a table of signed short. -'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells. -The return value is tableLog if everything proceeded as expected. -It is 0 if there is a single symbol within distribution. -If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using LIZFSE_isError()). - -'normalizedCounter' can be saved in a compact manner to a memory area using LIZFSE_writeNCount(). -'buffer' must be already allocated. -For guaranteed success, buffer size must be at least LIZFSE_headerBound(). -The result of the function is the number of bytes written into 'buffer'. -If there is an error, the function will return an ErrorCode (which can be tested using LIZFSE_isError(); ex : buffer size too small). - -'normalizedCounter' can then be used to create the compression table 'CTable'. -The space required by 'CTable' must be already allocated, using LIZFSE_createCTable(). -You can then use LIZFSE_buildCTable() to fill 'CTable'. -If there is an error, both functions will return an ErrorCode (which can be tested using LIZFSE_isError()). - -'CTable' can then be used to compress 'src', with LIZFSE_compress_usingCTable(). -Similar to LIZFSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize' -The function returns the size of compressed data (without header), necessarily <= `dstCapacity`. -If it returns '0', compressed data could not fit into 'dst'. -If there is an error, the function will return an ErrorCode (which can be tested using LIZFSE_isError()). -*/ - - -/* *** DECOMPRESSION *** */ - -/*! LIZFSE_readNCount(): - Read compactly saved 'normalizedCounter' from 'rBuffer'. - @return : size read from 'rBuffer', - or an errorCode, which can be tested using LIZFSE_isError(). - maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ -size_t LIZFSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize); - -/*! Constructor and Destructor of LIZFSE_DTable. - Note that its size depends on 'tableLog' */ -typedef unsigned LIZFSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ -LIZFSE_DTable* LIZFSE_createDTable(unsigned tableLog); -void LIZFSE_freeDTable(LIZFSE_DTable* dt); - -/*! LIZFSE_buildDTable(): - Builds 'dt', which must be already allocated, using LIZFSE_createDTable(). - return : 0, or an errorCode, which can be tested using LIZFSE_isError() */ -size_t LIZFSE_buildDTable (LIZFSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); - -/*! LIZFSE_decompress_usingDTable(): - Decompress compressed source `cSrc` of size `cSrcSize` using `dt` - into `dst` which must be already allocated. - @return : size of regenerated data (necessarily <= `dstCapacity`), - or an errorCode, which can be tested using LIZFSE_isError() */ -size_t LIZFSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const LIZFSE_DTable* dt); - -/*! -Tutorial : ----------- -(Note : these functions only decompress FSE-compressed blocks. - If block is uncompressed, use memcpy() instead - If block is a single repeated byte, use memset() instead ) - -The first step is to obtain the normalized frequencies of symbols. -This can be performed by LIZFSE_readNCount() if it was saved using LIZFSE_writeNCount(). -'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short. -In practice, that means it's necessary to know 'maxSymbolValue' beforehand, -or size the table to handle worst case situations (typically 256). -LIZFSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'. -The result of LIZFSE_readNCount() is the number of bytes read from 'rBuffer'. -Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that. -If there is an error, the function will return an error code, which can be tested using LIZFSE_isError(). - -The next step is to build the decompression tables 'LIZFSE_DTable' from 'normalizedCounter'. -This is performed by the function LIZFSE_buildDTable(). -The space required by 'LIZFSE_DTable' must be already allocated using LIZFSE_createDTable(). -If there is an error, the function will return an error code, which can be tested using LIZFSE_isError(). - -`LIZFSE_DTable` can then be used to decompress `cSrc`, with LIZFSE_decompress_usingDTable(). -`cSrcSize` must be strictly correct, otherwise decompression will fail. -LIZFSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`). -If there is an error, the function will return an error code, which can be tested using LIZFSE_isError(). (ex: dst buffer too small) -*/ - - -#ifdef LIZFSE_STATIC_LINKING_ONLY - -/* *** Dependency *** */ -#include "bitstream.h" - - -/* ***************************************** -* Static allocation -*******************************************/ -/* FSE buffer bounds */ -#define LIZFSE_NCOUNTBOUND 512 -#define LIZFSE_BLOCKBOUND(size) (size + (size>>7)) -#define LIZFSE_COMPRESSBOUND(size) (LIZFSE_NCOUNTBOUND + LIZFSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ - -/* It is possible to statically allocate FSE CTable/DTable as a table of unsigned using below macros */ -#define LIZFSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2)) -#define LIZFSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<= BIT_DStream_completed - -When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. -Checking if DStream has reached its end is performed by : - BIT_endOfDStream(&DStream); -Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible. - LIZFSE_endOfDState(&DState); -*/ - - -/* ***************************************** -* FSE unsafe API -*******************************************/ -static unsigned char LIZFSE_decodeSymbolFast(LIZFSE_DState_t* DStatePtr, BIT_DStream_t* bitD); -/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ - - -/* ***************************************** -* Implementation of inlined functions -*******************************************/ -typedef struct { - int deltaFindState; - U32 deltaNbBits; -} LIZFSE_symbolCompressionTransform; /* total 8 bytes */ - -MEM_STATIC void LIZFSE_initCState(LIZFSE_CState_t* statePtr, const LIZFSE_CTable* ct) -{ - const void* ptr = ct; - const U16* u16ptr = (const U16*) ptr; - const U32 tableLog = MEM_read16(ptr); - statePtr->value = (ptrdiff_t)1<stateTable = u16ptr+2; - statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1)); - statePtr->stateLog = tableLog; -} - - -/*! LIZFSE_initCState2() : -* Same as LIZFSE_initCState(), but the first symbol to include (which will be the last to be read) -* uses the smallest state value possible, saving the cost of this symbol */ -MEM_STATIC void LIZFSE_initCState2(LIZFSE_CState_t* statePtr, const LIZFSE_CTable* ct, U32 symbol) -{ - LIZFSE_initCState(statePtr, ct); - { const LIZFSE_symbolCompressionTransform symbolTT = ((const LIZFSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; - const U16* stateTable = (const U16*)(statePtr->stateTable); - U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16); - statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits; - statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; - } -} - -MEM_STATIC void LIZFSE_encodeSymbol(BIT_CStream_t* bitC, LIZFSE_CState_t* statePtr, U32 symbol) -{ - const LIZFSE_symbolCompressionTransform symbolTT = ((const LIZFSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; - const U16* const stateTable = (const U16*)(statePtr->stateTable); - U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); - BIT_addBits(bitC, statePtr->value, nbBitsOut); - statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; -} - -MEM_STATIC void LIZFSE_flushCState(BIT_CStream_t* bitC, const LIZFSE_CState_t* statePtr) -{ - BIT_addBits(bitC, statePtr->value, statePtr->stateLog); - BIT_flushBits(bitC); -} - -/* ====== Decompression ====== */ - -typedef struct { - U16 tableLog; - U16 fastMode; -} LIZFSE_DTableHeader; /* sizeof U32 */ - -typedef struct -{ - unsigned short newState; - unsigned char symbol; - unsigned char nbBits; -} LIZFSE_decode_t; /* size == U32 */ - -MEM_STATIC void LIZFSE_initDState(LIZFSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const LIZFSE_DTable* dt) -{ - const void* ptr = dt; - const LIZFSE_DTableHeader* const DTableH = (const LIZFSE_DTableHeader*)ptr; - DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); - BIT_reloadDStream(bitD); - DStatePtr->table = dt + 1; -} - -MEM_STATIC BYTE LIZFSE_peekSymbol(const LIZFSE_DState_t* DStatePtr) -{ - LIZFSE_decode_t const DInfo = ((const LIZFSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; - return DInfo.symbol; -} - -MEM_STATIC void LIZFSE_updateState(LIZFSE_DState_t* DStatePtr, BIT_DStream_t* bitD) -{ - LIZFSE_decode_t const DInfo = ((const LIZFSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; - U32 const nbBits = DInfo.nbBits; - size_t const lowBits = BIT_readBits(bitD, nbBits); - DStatePtr->state = DInfo.newState + lowBits; -} - -MEM_STATIC BYTE LIZFSE_decodeSymbol(LIZFSE_DState_t* DStatePtr, BIT_DStream_t* bitD) -{ - LIZFSE_decode_t const DInfo = ((const LIZFSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; - U32 const nbBits = DInfo.nbBits; - BYTE const symbol = DInfo.symbol; - size_t const lowBits = BIT_readBits(bitD, nbBits); - - DStatePtr->state = DInfo.newState + lowBits; - return symbol; -} - -/*! LIZFSE_decodeSymbolFast() : - unsafe, only works if no symbol has a probability > 50% */ -MEM_STATIC BYTE LIZFSE_decodeSymbolFast(LIZFSE_DState_t* DStatePtr, BIT_DStream_t* bitD) -{ - LIZFSE_decode_t const DInfo = ((const LIZFSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; - U32 const nbBits = DInfo.nbBits; - BYTE const symbol = DInfo.symbol; - size_t const lowBits = BIT_readBitsFast(bitD, nbBits); - - DStatePtr->state = DInfo.newState + lowBits; - return symbol; -} - -MEM_STATIC unsigned LIZFSE_endOfDState(const LIZFSE_DState_t* DStatePtr) -{ - return DStatePtr->state == 0; -} - - - -#ifndef LIZFSE_COMMONDEFS_ONLY - -/* ************************************************************** -* Tuning parameters -****************************************************************/ -/*!MEMORY_USAGE : -* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) -* Increasing memory usage improves compression ratio -* Reduced memory usage can improve speed, due to cache effect -* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ -#define LIZFSE_MAX_MEMORY_USAGE 14 -#define LIZFSE_DEFAULT_MEMORY_USAGE 13 - -/*!LIZFSE_MAX_SYMBOL_VALUE : -* Maximum symbol value authorized. -* Required for proper stack allocation */ -#define LIZFSE_MAX_SYMBOL_VALUE 255 - - -/* ************************************************************** -* template functions type & suffix -****************************************************************/ -#define LIZFSE_FUNCTION_TYPE BYTE -#define LIZFSE_FUNCTION_EXTENSION -#define LIZFSE_DECODE_TYPE LIZFSE_decode_t - - -#endif /* !LIZFSE_COMMONDEFS_ONLY */ - - -/* *************************************************************** -* Constants -*****************************************************************/ -#define LIZFSE_MAX_TABLELOG (LIZFSE_MAX_MEMORY_USAGE-2) -#define LIZFSE_MAX_TABLESIZE (1U< LIZFSE_TABLELOG_ABSOLUTE_MAX -# error "LIZFSE_MAX_TABLELOG > LIZFSE_TABLELOG_ABSOLUTE_MAX is not supported" -#endif - -#define LIZFSE_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3) - - -#endif /* LIZFSE_STATIC_LINKING_ONLY */ - - -#if defined (__cplusplus) -} -#endif - -#endif /* LIZFSE_H */ diff --git a/C/lizard/huf.h b/C/lizard/huf.h deleted file mode 100644 index eb2b1375..00000000 --- a/C/lizard/huf.h +++ /dev/null @@ -1,228 +0,0 @@ -/* ****************************************************************** - Huffman coder, part of New Generation Entropy library - header file - Copyright (C) 2013-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Source repository : https://github.com/Cyan4973/FiniteStateEntropy -****************************************************************** */ -#ifndef LIZHUF_H_298734234 -#define LIZHUF_H_298734234 - -#if defined (__cplusplus) -extern "C" { -#endif - - -/* *** Dependencies *** */ -#include /* size_t */ - - -/* *** simple functions *** */ -/** -LIZHUF_compress() : - Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'. - 'dst' buffer must be already allocated. - Compression runs faster if `dstCapacity` >= LIZHUF_compressBound(srcSize). - `srcSize` must be <= `LIZHUF_BLOCKSIZE_MAX` == 128 KB. - @return : size of compressed data (<= `dstCapacity`). - Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! - if return == 1, srcData is a single repeated byte symbol (RLE compression). - if LIZHUF_isError(return), compression failed (more details using LIZHUF_getErrorName()) -*/ -size_t LIZHUF_compress(void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - -/** -LIZHUF_decompress() : - Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', - into already allocated buffer 'dst', of minimum size 'dstSize'. - `dstSize` : **must** be the ***exact*** size of original (uncompressed) data. - Note : in contrast with FSE, LIZHUF_decompress can regenerate - RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, - because it knows size to regenerate. - @return : size of regenerated data (== dstSize), - or an error code, which can be tested using LIZHUF_isError() -*/ -size_t LIZHUF_decompress(void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize); - - -/* **************************************** -* Tool functions -******************************************/ -#define LIZHUF_BLOCKSIZE_MAX (128 * 1024) -size_t LIZHUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ - -/* Error Management */ -unsigned LIZHUF_isError(size_t code); /**< tells if a return value is an error code */ -const char* LIZHUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ - - -/* *** Advanced function *** */ - -/** LIZHUF_compress2() : -* Same as LIZHUF_compress(), but offers direct control over `maxSymbolValue` and `tableLog` */ -size_t LIZHUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); - - -#ifdef LIZHUF_STATIC_LINKING_ONLY - -/* *** Dependencies *** */ -#include "mem.h" /* U32 */ - - -/* *** Constants *** */ -#define LIZHUF_TABLELOG_ABSOLUTEMAX 16 /* absolute limit of LIZHUF_MAX_TABLELOG. Beyond that value, code does not work */ -#define LIZHUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to LIZHUF_ABSOLUTEMAX_TABLELOG */ -#define LIZHUF_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */ -#define LIZHUF_SYMBOLVALUE_MAX 255 -#if (LIZHUF_TABLELOG_MAX > LIZHUF_TABLELOG_ABSOLUTEMAX) -# error "LIZHUF_TABLELOG_MAX is too large !" -#endif - - -/* **************************************** -* Static allocation -******************************************/ -/* HUF buffer bounds */ -#define LIZHUF_CTABLEBOUND 129 -#define LIZHUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */ -#define LIZHUF_COMPRESSBOUND(size) (LIZHUF_CTABLEBOUND + LIZHUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ - -/* static allocation of HUF's Compression Table */ -#define LIZHUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ - U32 name##hb[maxSymbolValue+1]; \ - void* name##hv = &(name##hb); \ - LIZHUF_CElt* name = (LIZHUF_CElt*)(name##hv) /* no final ; */ - -/* static allocation of HUF's DTable */ -typedef U32 LIZHUF_DTable; -#define LIZHUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog))) -#define LIZHUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \ - LIZHUF_DTable DTable[LIZHUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1)*0x1000001) } -#define LIZHUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \ - LIZHUF_DTable DTable[LIZHUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog)*0x1000001) } - - -/* **************************************** -* Advanced decompression functions -******************************************/ -size_t LIZHUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -size_t LIZHUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ - -size_t LIZHUF_decompress4X_DCtx (LIZHUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */ -size_t LIZHUF_decompress4X_hufOnly(LIZHUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */ -size_t LIZHUF_decompress4X2_DCtx(LIZHUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -size_t LIZHUF_decompress4X4_DCtx(LIZHUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ - -size_t LIZHUF_decompress1X_DCtx (LIZHUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); -size_t LIZHUF_decompress1X2_DCtx(LIZHUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -size_t LIZHUF_decompress1X4_DCtx(LIZHUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ - - -/* **************************************** -* HUF detailed API -******************************************/ -/*! -LIZHUF_compress() does the following: -1. count symbol occurrence from source[] into table count[] using LIZFSE_count() -2. (optional) refine tableLog using LIZHUF_optimalTableLog() -3. build Huffman table from count using LIZHUF_buildCTable() -4. save Huffman table to memory buffer using LIZHUF_writeCTable() -5. encode the data stream using LIZHUF_compress4X_usingCTable() - -The following API allows targeting specific sub-functions for advanced tasks. -For example, it's possible to compress several blocks using the same 'CTable', -or to save and regenerate 'CTable' using external methods. -*/ -/* LIZFSE_count() : find it within "fse.h" */ -unsigned LIZHUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); -typedef struct LIZHUF_CElt_s LIZHUF_CElt; /* incomplete type */ -size_t LIZHUF_buildCTable (LIZHUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); -size_t LIZHUF_writeCTable (void* dst, size_t maxDstSize, const LIZHUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog); -size_t LIZHUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const LIZHUF_CElt* CTable); - - -/*! LIZHUF_readStats() : - Read compact Huffman tree, saved by LIZHUF_writeCTable(). - `huffWeight` is destination buffer. - @return : size read from `src` , or an error Code . - Note : Needed by LIZHUF_readCTable() and LIZHUF_readDTableXn() . */ -size_t LIZHUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, - U32* nbSymbolsPtr, U32* tableLogPtr, - const void* src, size_t srcSize); - -/** LIZHUF_readCTable() : -* Loading a CTable saved with LIZHUF_writeCTable() */ -size_t LIZHUF_readCTable (LIZHUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize); - - -/* -LIZHUF_decompress() does the following: -1. select the decompression algorithm (X2, X4) based on pre-computed heuristics -2. build Huffman table from save, using LIZHUF_readDTableXn() -3. decode 1 or 4 segments in parallel using LIZHUF_decompressSXn_usingDTable -*/ - -/** LIZHUF_selectDecoder() : -* Tells which decoder is likely to decode faster, -* based on a set of pre-determined metrics. -* @return : 0==LIZHUF_decompress4X2, 1==LIZHUF_decompress4X4 . -* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ -U32 LIZHUF_selectDecoder (size_t dstSize, size_t cSrcSize); - -size_t LIZHUF_readDTableX2 (LIZHUF_DTable* DTable, const void* src, size_t srcSize); -size_t LIZHUF_readDTableX4 (LIZHUF_DTable* DTable, const void* src, size_t srcSize); - -size_t LIZHUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const LIZHUF_DTable* DTable); -size_t LIZHUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const LIZHUF_DTable* DTable); -size_t LIZHUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const LIZHUF_DTable* DTable); - - -/* single stream variants */ - -size_t LIZHUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); -size_t LIZHUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const LIZHUF_CElt* CTable); - -size_t LIZHUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ -size_t LIZHUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ - -size_t LIZHUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const LIZHUF_DTable* DTable); -size_t LIZHUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const LIZHUF_DTable* DTable); -size_t LIZHUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const LIZHUF_DTable* DTable); - - -#endif /* LIZHUF_STATIC_LINKING_ONLY */ - - -#if defined (__cplusplus) -} -#endif - -#endif /* LIZHUF_H_298734234 */ diff --git a/C/lizard/liz_common.h b/C/lizard/liz_common.h deleted file mode 100644 index 0f7585ed..00000000 --- a/C/lizard/liz_common.h +++ /dev/null @@ -1,497 +0,0 @@ -/* - LZ5 - Fast LZ compression algorithm - Copyright (C) 2011-2015, Yann Collet. - Copyright (C) 2015-2016, Przemyslaw Skibinski - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ5 source repository : https://github.com/inikep/lz5 -*/ - -#ifndef LIZ_COMMON_H_2983 -#define LIZ_COMMON_H_2983 - -#if defined (__cplusplus) -extern "C" { -#endif - - -/*-************************************ -* Memory routines -**************************************/ -#include /* malloc, calloc, free */ -#include /* memset, memcpy */ -#include /* intptr_t */ -#include "mem.h" -#include "liz_compress.h" /* LIZ_GCC_VERSION */ - -//#define LIZ_USE_LOGS -#define LIZ_LOG_COMPRESS(...) //printf(__VA_ARGS__) -#define LIZ_LOG_DECOMPRESS(...) //printf(__VA_ARGS__) - -#define LIZ_LOG_COMPRESS_LZ4(...) //printf(__VA_ARGS__) -#define COMPLOG_CODEWORDS_LZ4(...) //printf(__VA_ARGS__) -#define LIZ_LOG_DECOMPRESS_LZ4(...) //printf(__VA_ARGS__) -#define DECOMPLOG_CODEWORDS_LZ4(...) //printf(__VA_ARGS__) - -#define LIZ_LOG_COMPRESS_LZ5v2(...) //printf(__VA_ARGS__) -#define COMPLOG_CODEWORDS_LZ5v2(...) //printf(__VA_ARGS__) -#define LIZ_LOG_DECOMPRESS_LZ5v2(...) //printf(__VA_ARGS__) -#define DECOMPLOG_CODEWORDS_LZ5v2(...) //printf(__VA_ARGS__) - - - - -/*-************************************ -* Common Constants -**************************************/ -#define MINMATCH 4 -//#define USE_LZ4_ONLY -//#define LIZ_USE_TEST - -#define LIZ_DICT_SIZE (1<<24) -#define WILDCOPYLENGTH 16 -#define LASTLITERALS WILDCOPYLENGTH -#define MFLIMIT (WILDCOPYLENGTH+MINMATCH) - -#define LIZ_MAX_PRICE (1<<28) -#define LIZ_INIT_LAST_OFFSET 0 -#define LIZ_MAX_16BIT_OFFSET (1<<16) -#define MM_LONGOFF 16 -#define LIZ_BLOCK_SIZE_PAD (LIZ_BLOCK_SIZE+32) -#define LIZ_COMPRESS_ADD_BUF (5*LIZ_BLOCK_SIZE_PAD) -#ifndef LIZ_NO_HUFFMAN - #define LIZ_COMPRESS_ADD_HUF LIZHUF_compressBound(LIZ_BLOCK_SIZE_PAD) - #define LIZ_LIZHUF_BLOCK_SIZE LIZ_BLOCK_SIZE -#else - #define LIZ_COMPRESS_ADD_HUF 0 - #define LIZ_LIZHUF_BLOCK_SIZE 1 -#endif - -/* LZ4 codewords */ -#define ML_BITS_LZ4 4 -#define ML_MASK_LZ4 ((1U<= 1<<16 */ - U32 sufficientLength; /* used only by optimal parser: size of matches which is acceptable: larger == more compression, slower */ - U32 fullSearch; /* used only by optimal parser: perform full search of matches: 1 == more compression, slower */ - LIZ_parser_type parserType; - LIZ_decompress_type decompressType; -} LIZ_parameters; - - -struct LIZ_stream_s -{ - const BYTE* end; /* next block here to continue on current prefix */ - const BYTE* base; /* All index relative to this position */ - const BYTE* dictBase; /* alternate base for extDict */ - U32 dictLimit; /* below that point, need extDict */ - U32 lowLimit; /* below that point, no more dict */ - U32 nextToUpdate; /* index from which to continue dictionary update */ - U32 allocatedMemory; - int compressionLevel; - LIZ_parameters params; - U32 hashTableSize; - U32 chainTableSize; - U32* chainTable; - U32* hashTable; - int last_off; - const BYTE* off24pos; - U32 huffType; - U32 comprStreamLen; - - BYTE* huffBase; - BYTE* huffEnd; - BYTE* offset16Base; - BYTE* offset24Base; - BYTE* lenBase; - BYTE* literalsBase; - BYTE* flagsBase; - BYTE* offset16Ptr; - BYTE* offset24Ptr; - BYTE* lenPtr; - BYTE* literalsPtr; - BYTE* flagsPtr; - BYTE* offset16End; - BYTE* offset24End; - BYTE* lenEnd; - BYTE* literalsEnd; - BYTE* flagsEnd; - U32 flagFreq[256]; - U32 litFreq[256]; - U32 litSum, flagSum; - U32 litPriceSum, log2LitSum, log2FlagSum; - U32 cachedPrice; - U32 cachedLitLength; - const BYTE* cachedLiterals; - const BYTE* diffBase; - const BYTE* srcBase; - const BYTE* destBase; -}; - -struct LIZ_dstream_s -{ - const BYTE* offset16Ptr; - const BYTE* offset24Ptr; - const BYTE* lenPtr; - const BYTE* literalsPtr; - const BYTE* flagsPtr; - const BYTE* offset16End; - const BYTE* offset24End; - const BYTE* lenEnd; - const BYTE* literalsEnd; - const BYTE* flagsEnd; - const BYTE* diffBase; - intptr_t last_off; -}; - -typedef struct LIZ_dstream_s LIZ_dstream_t; - -/* ************************************* -* HC Pre-defined compression levels -***************************************/ -#define LIZ_WINDOWLOG_LZ4 16 -#define LIZ_CHAINLOG_LZ4 LIZ_WINDOWLOG_LZ4 -#define LIZ_HASHLOG_LZ4 18 -#define LIZ_HASHLOG_LZ4SM 12 - -#define LIZ_WINDOWLOG_LZ5v2 22 -#define LIZ_CHAINLOG_LZ5v2 LIZ_WINDOWLOG_LZ5v2 -#define LIZ_HASHLOG_LZ5v2 18 - - - -static const LIZ_parameters LIZ_defaultParameters[LIZ_MAX_CLEVEL+1-LIZ_MIN_CLEVEL] = -{ - /* windLog, contentLog, HashLog, H3, Snum, SL, MMLongOff, SuffL, FS, Parser function, Decompressor type */ - { LIZ_WINDOWLOG_LZ4, 0, LIZ_HASHLOG_LZ4SM, 0, 0, 0, 0, 0, 0, LIZ_parser_fastSmall, LIZ_coderwords_LZ4 }, // level 10 - { LIZ_WINDOWLOG_LZ4, 0, LIZ_HASHLOG_LZ4, 0, 0, 0, 0, 0, 0, LIZ_parser_fast, LIZ_coderwords_LZ4 }, // level 11 - { LIZ_WINDOWLOG_LZ4, 0, LIZ_HASHLOG_LZ4, 0, 0, 0, 0, 0, 0, LIZ_parser_noChain, LIZ_coderwords_LZ4 }, // level 12 - { LIZ_WINDOWLOG_LZ4, LIZ_CHAINLOG_LZ4, LIZ_HASHLOG_LZ4, 0, 2, 5, 0, 0, 0, LIZ_parser_hashChain, LIZ_coderwords_LZ4 }, // level 13 - { LIZ_WINDOWLOG_LZ4, LIZ_CHAINLOG_LZ4, LIZ_HASHLOG_LZ4, 0, 4, 5, 0, 0, 0, LIZ_parser_hashChain, LIZ_coderwords_LZ4 }, // level 14 - { LIZ_WINDOWLOG_LZ4, LIZ_CHAINLOG_LZ4, LIZ_HASHLOG_LZ4, 0, 8, 5, 0, 0, 0, LIZ_parser_hashChain, LIZ_coderwords_LZ4 }, // level 15 - { LIZ_WINDOWLOG_LZ4, LIZ_CHAINLOG_LZ4, LIZ_HASHLOG_LZ4, 0, 16, 4, 0, 0, 0, LIZ_parser_hashChain, LIZ_coderwords_LZ4 }, // level 16 - { LIZ_WINDOWLOG_LZ4, LIZ_CHAINLOG_LZ4, LIZ_HASHLOG_LZ4, 0, 256, 4, 0, 0, 0, LIZ_parser_hashChain, LIZ_coderwords_LZ4 }, // level 17 - { LIZ_WINDOWLOG_LZ4, LIZ_WINDOWLOG_LZ4+1, LIZ_HASHLOG_LZ4, 16, 16, 4, 0, 1<<10, 1, LIZ_parser_optimalPriceBT, LIZ_coderwords_LZ4 }, // level 18 - { LIZ_WINDOWLOG_LZ4, LIZ_WINDOWLOG_LZ4+1, 23, 16, 256, 4, 0, 1<<10, 1, LIZ_parser_optimalPriceBT, LIZ_coderwords_LZ4 }, // level 19 - /* windLog, contentLog, HashLog, H3, Snum, SL, MMLongOff, SuffL, FS, Parser function, Decompressor type */ - { LIZ_WINDOWLOG_LZ5v2, 0, 14, 0, 1, 5, MM_LONGOFF, 0, 0, LIZ_parser_fastBig, LIZ_coderwords_LZ5v2 }, // level 20 - { LIZ_WINDOWLOG_LZ5v2, LIZ_CHAINLOG_LZ5v2, 14, 13, 1, 5, MM_LONGOFF, 0, 0, LIZ_parser_priceFast, LIZ_coderwords_LZ5v2 }, // level 21 - { LIZ_WINDOWLOG_LZ5v2, LIZ_CHAINLOG_LZ5v2, LIZ_HASHLOG_LZ5v2, 13, 1, 5, MM_LONGOFF, 0, 0, LIZ_parser_priceFast, LIZ_coderwords_LZ5v2 }, // level 22 - { LIZ_WINDOWLOG_LZ5v2, LIZ_CHAINLOG_LZ5v2, LIZ_HASHLOG_LZ5v2, 13, 1, 5, MM_LONGOFF, 64, 0, LIZ_parser_lowestPrice, LIZ_coderwords_LZ5v2 }, // level 23 - { LIZ_WINDOWLOG_LZ5v2, LIZ_CHAINLOG_LZ5v2, 23, 16, 2, 5, MM_LONGOFF, 64, 0, LIZ_parser_lowestPrice, LIZ_coderwords_LZ5v2 }, // level 24 - { LIZ_WINDOWLOG_LZ5v2, LIZ_CHAINLOG_LZ5v2, 23, 16, 8, 4, MM_LONGOFF, 64, 0, LIZ_parser_lowestPrice, LIZ_coderwords_LZ5v2 }, // level 25 - { LIZ_WINDOWLOG_LZ5v2, LIZ_CHAINLOG_LZ5v2+1, 23, 16, 8, 4, MM_LONGOFF, 64, 1, LIZ_parser_optimalPriceBT, LIZ_coderwords_LZ5v2 }, // level 26 - { LIZ_WINDOWLOG_LZ5v2, LIZ_CHAINLOG_LZ5v2+1, 23, 16, 128, 4, MM_LONGOFF, 64, 1, LIZ_parser_optimalPriceBT, LIZ_coderwords_LZ5v2 }, // level 27 - { LIZ_WINDOWLOG_LZ5v2, LIZ_CHAINLOG_LZ5v2+1, 23, 24, 1<<10, 4, MM_LONGOFF, 1<<10, 1, LIZ_parser_optimalPriceBT, LIZ_coderwords_LZ5v2 }, // level 28 - { 24, 25, 23, 24, 1<<10, 4, MM_LONGOFF, 1<<10, 1, LIZ_parser_optimalPriceBT, LIZ_coderwords_LZ5v2 }, // level 29 -#ifndef LIZ_NO_HUFFMAN - /* windLog, contentLog, HashLog, H3, Snum, SL, MMLongOff, SuffL, FS, Parser function, Decompressor type */ - { LIZ_WINDOWLOG_LZ4, 0, LIZ_HASHLOG_LZ4SM, 0, 0, 0, 0, 0, 0, LIZ_parser_fastSmall, LIZ_coderwords_LZ4 }, // level 30 - { LIZ_WINDOWLOG_LZ4, 0, LIZ_HASHLOG_LZ4, 0, 0, 0, 0, 0, 0, LIZ_parser_fast, LIZ_coderwords_LZ4 }, // level 31 - { LIZ_WINDOWLOG_LZ4, 0, 14, 0, 0, 0, 0, 0, 0, LIZ_parser_noChain, LIZ_coderwords_LZ4 }, // level 32 - { LIZ_WINDOWLOG_LZ4, 0, LIZ_HASHLOG_LZ4, 0, 0, 0, 0, 0, 0, LIZ_parser_noChain, LIZ_coderwords_LZ4 }, // level 33 - { LIZ_WINDOWLOG_LZ4, LIZ_CHAINLOG_LZ4, LIZ_HASHLOG_LZ4, 0, 2, 5, 0, 0, 0, LIZ_parser_hashChain, LIZ_coderwords_LZ4 }, // level 34 - { LIZ_WINDOWLOG_LZ4, LIZ_CHAINLOG_LZ4, LIZ_HASHLOG_LZ4, 0, 4, 5, 0, 0, 0, LIZ_parser_hashChain, LIZ_coderwords_LZ4 }, // level 35 - { LIZ_WINDOWLOG_LZ4, LIZ_CHAINLOG_LZ4, LIZ_HASHLOG_LZ4, 0, 8, 5, 0, 0, 0, LIZ_parser_hashChain, LIZ_coderwords_LZ4 }, // level 36 - { LIZ_WINDOWLOG_LZ4, LIZ_CHAINLOG_LZ4, LIZ_HASHLOG_LZ4, 0, 16, 4, 0, 0, 0, LIZ_parser_hashChain, LIZ_coderwords_LZ4 }, // level 37 - { LIZ_WINDOWLOG_LZ4, LIZ_CHAINLOG_LZ4, LIZ_HASHLOG_LZ4, 0, 256, 4, 0, 0, 0, LIZ_parser_hashChain, LIZ_coderwords_LZ4 }, // level 38 - { LIZ_WINDOWLOG_LZ4, LIZ_WINDOWLOG_LZ4+1, 23, 16, 256, 4, 0, 1<<10, 1, LIZ_parser_optimalPriceBT, LIZ_coderwords_LZ4 }, // level 39 - /* windLog, contentLog, HashLog, H3, Snum, SL, MMLongOff, SuffL, FS, Parser function, Decompressor type */ - { LIZ_WINDOWLOG_LZ5v2, 0, 14, 0, 1, 5, MM_LONGOFF, 0, 0, LIZ_parser_fastBig, LIZ_coderwords_LZ5v2 }, // level 40 - { LIZ_WINDOWLOG_LZ5v2, LIZ_CHAINLOG_LZ5v2, 14, 13, 1, 5, MM_LONGOFF, 0, 0, LIZ_parser_priceFast, LIZ_coderwords_LZ5v2 }, // level 41 - { LIZ_WINDOWLOG_LZ5v2, LIZ_CHAINLOG_LZ5v2, LIZ_HASHLOG_LZ5v2, 13, 1, 5, MM_LONGOFF, 0, 0, LIZ_parser_priceFast, LIZ_coderwords_LZ5v2 }, // level 42 - { LIZ_WINDOWLOG_LZ5v2, LIZ_CHAINLOG_LZ5v2, LIZ_HASHLOG_LZ5v2, 13, 1, 5, MM_LONGOFF, 64, 0, LIZ_parser_lowestPrice, LIZ_coderwords_LZ5v2 }, // level 43 - { LIZ_WINDOWLOG_LZ5v2, LIZ_CHAINLOG_LZ5v2, 23, 16, 2, 5, MM_LONGOFF, 64, 0, LIZ_parser_lowestPrice, LIZ_coderwords_LZ5v2 }, // level 44 - { LIZ_WINDOWLOG_LZ5v2, LIZ_CHAINLOG_LZ5v2, 23, 16, 8, 4, MM_LONGOFF, 64, 0, LIZ_parser_lowestPrice, LIZ_coderwords_LZ5v2 }, // level 45 - { LIZ_WINDOWLOG_LZ5v2, LIZ_CHAINLOG_LZ5v2, 23, 16, 8, 4, MM_LONGOFF, 64, 0, LIZ_parser_optimalPrice, LIZ_coderwords_LZ5v2 }, // level 46 - { LIZ_WINDOWLOG_LZ5v2, LIZ_CHAINLOG_LZ5v2+1, 23, 16, 8, 4, MM_LONGOFF, 64, 1, LIZ_parser_optimalPriceBT, LIZ_coderwords_LZ5v2 }, // level 47 - { LIZ_WINDOWLOG_LZ5v2, LIZ_CHAINLOG_LZ5v2+1, 23, 16, 128, 4, MM_LONGOFF, 64, 1, LIZ_parser_optimalPriceBT, LIZ_coderwords_LZ5v2 }, // level 48 - { 24, 25, 23, 24, 1<<10, 4, MM_LONGOFF, 1<<10, 1, LIZ_parser_optimalPriceBT, LIZ_coderwords_LZ5v2 }, // level 49 -#endif -// { 10, 10, 10, 0, 0, 4, 0, 0, 0, LIZ_fast }, // min values -// { 24, 24, 28, 24, 1<<24, 7, 0, 1<<24, 2, LIZ_optimal_price }, // max values -}; - - - -/*-************************************ -* Compiler Options -**************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline -# include -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ -#else -# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -# if defined(__GNUC__) || defined(__clang__) -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -# else -# define FORCE_INLINE static -# endif /* __STDC_VERSION__ */ -#endif /* _MSC_VER */ - -#define LIZ_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -#if (LIZ_GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__) -# define expect(expr,value) (__builtin_expect ((expr),(value)) ) -#else -# define expect(expr,value) (expr) -#endif - -#define likely(expr) expect((expr) != 0, 1) -#define unlikely(expr) expect((expr) != 0, 0) - -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -#define ALLOCATOR(n,s) calloc(n,s) -#define FREEMEM free -#define MEM_INIT memset -#ifndef MAX - #define MAX(a,b) ((a)>(b))?(a):(b) -#endif -#ifndef MIN - #define MIN(a,b) ((a)<(b)?(a):(b)) -#endif - -#if MINMATCH == 3 - #define MEM_readMINMATCH(ptr) (U32)(MEM_read32(ptr)<<8) -#else - #define MEM_readMINMATCH(ptr) (U32)(MEM_read32(ptr)) -#endif - - - - -/*-************************************ -* Reading and writing into memory -**************************************/ -#define STEPSIZE sizeof(size_t) - - -MEM_STATIC void LIZ_copy8(void* dst, const void* src) -{ - memcpy(dst,src,8); -} - -/* customized variant of memcpy, which can overwrite up to 7 bytes beyond dstEnd */ -MEM_STATIC void LIZ_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) -{ - BYTE* d = (BYTE*)dstPtr; - const BYTE* s = (const BYTE*)srcPtr; - BYTE* const e = (BYTE*)dstEnd; - -#if 0 - const size_t l2 = 8 - (((size_t)d) & (sizeof(void*)-1)); - LIZ_copy8(d,s); if (d>e-9) return; - d+=l2; s+=l2; -#endif /* join to align */ - - do { LIZ_copy8(d,s); d+=8; s+=8; } while (d= 3) /* GCC Intrinsic */ - return 31 - __builtin_clz(val); -# else /* Software version */ - static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; - U32 v = val; - int r; - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27]; - return r; -# endif -} - - -/*-************************************ -* Common functions -**************************************/ -MEM_STATIC unsigned LIZ_NbCommonBytes (register size_t val) -{ - if (MEM_isLittleEndian()) { - if (MEM_64bits()) { -# if defined(_MSC_VER) && defined(_WIN64) && !defined(LIZ_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanForward64( &r, (U64)val ); - return (int)(r>>3); -# elif (defined(__clang__) || (LIZ_GCC_VERSION >= 304)) && !defined(LIZ_FORCE_SW_BITCOUNT) - return (__builtin_ctzll((U64)val) >> 3); -# else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; - return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; -# endif - } else /* 32 bits */ { -# if defined(_MSC_VER) && !defined(LIZ_FORCE_SW_BITCOUNT) - unsigned long r; - _BitScanForward( &r, (U32)val ); - return (int)(r>>3); -# elif (defined(__clang__) || (LIZ_GCC_VERSION >= 304)) && !defined(LIZ_FORCE_SW_BITCOUNT) - return (__builtin_ctz((U32)val) >> 3); -# else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; -# endif - } - } else /* Big Endian CPU */ { - if (MEM_64bits()) { -# if defined(_MSC_VER) && defined(_WIN64) && !defined(LIZ_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse64( &r, val ); - return (unsigned)(r>>3); -# elif (defined(__clang__) || (LIZ_GCC_VERSION >= 304)) && !defined(LIZ_FORCE_SW_BITCOUNT) - return (__builtin_clzll((U64)val) >> 3); -# else - unsigned r; - if (!(val>>32)) { r=4; } else { r=0; val>>=32; } - if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } - r += (!val); - return r; -# endif - } else /* 32 bits */ { -# if defined(_MSC_VER) && !defined(LIZ_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse( &r, (unsigned long)val ); - return (unsigned)(r>>3); -# elif (defined(__clang__) || (LIZ_GCC_VERSION >= 304)) && !defined(LIZ_FORCE_SW_BITCOUNT) - return (__builtin_clz((U32)val) >> 3); -# else - unsigned r; - if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } - r += (!val); - return r; -# endif - } - } -} - -MEM_STATIC unsigned LIZ_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) -{ - const BYTE* const pStart = pIn; - - while (likely(pIn - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ5 source repository : https://github.com/inikep/lz5 -*/ -#ifndef LIZ_H_2983 -#define LIZ_H_2983 - -#if defined (__cplusplus) -extern "C" { -#endif - -/* - * lz5_compress.h provides block compression functions. It gives full buffer control to user. - * Block compression functions are not-enough to send information, - * since it's still necessary to provide metadata (such as compressed size), - * and each application can do it in whichever way it wants. - * For interoperability, there is LZ5 frame specification (lz5_Frame_format.md). - * A library is provided to take care of it, see lz5frame.h. -*/ - - -/*^*************************************************************** -* Export parameters -*****************************************************************/ -/* -* LIZ_DLL_EXPORT : -* Enable exporting of functions when building a Windows DLL -*/ -#if defined(LIZ_DLL_EXPORT) && (LIZ_DLL_EXPORT==1) -# define LZ5LIB_API __declspec(dllexport) -#elif defined(LIZ_DLL_IMPORT) && (LIZ_DLL_IMPORT==1) -# define LZ5LIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ -#else -# define LZ5LIB_API -#endif - - -/*-************************************ -* Version -**************************************/ -#define LIZ_VERSION_MAJOR 2 /* for breaking interface changes */ -#define LIZ_VERSION_MINOR 0 /* for new (non-breaking) interface capabilities */ -#define LIZ_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ - -#define LIZ_VERSION_NUMBER (LIZ_VERSION_MAJOR *100*100 + LIZ_VERSION_MINOR *100 + LIZ_VERSION_RELEASE) -int LIZ_versionNumber (void); - -#define LIZ_LIB_VERSION LIZ_VERSION_MAJOR.LIZ_VERSION_MINOR.LIZ_VERSION_RELEASE -#define LIZ_QUOTE(str) #str -#define LIZ_EXPAND_AND_QUOTE(str) LIZ_QUOTE(str) -#define LIZ_VERSION_STRING LIZ_EXPAND_AND_QUOTE(LIZ_LIB_VERSION) -const char* LIZ_versionString (void); - -typedef struct LIZ_stream_s LIZ_stream_t; - -#define LIZ_MIN_CLEVEL 10 /* minimum compression level */ -#ifndef LIZ_NO_HUFFMAN - #define LIZ_MAX_CLEVEL 49 /* maximum compression level */ -#else - #define LIZ_MAX_CLEVEL 29 /* maximum compression level */ -#endif -#define LIZ_DEFAULT_CLEVEL 17 - - -/*-************************************ -* Simple Functions -**************************************/ - -LZ5LIB_API int LIZ_compress (const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); - -/* -LIZ_compress() : - Compresses 'sourceSize' bytes from buffer 'source' - into already allocated 'dest' buffer of size 'maxDestSize'. - Compression is guaranteed to succeed if 'maxDestSize' >= LIZ_compressBound(sourceSize). - It also runs faster, so it's a recommended setting. - If the function cannot compress 'source' into a more limited 'dest' budget, - compression stops *immediately*, and the function result is zero. - As a consequence, 'dest' content is not valid. - This function never writes outside 'dest' buffer, nor read outside 'source' buffer. - sourceSize : Max supported value is LIZ_MAX_INPUT_VALUE - maxDestSize : full or partial size of buffer 'dest' (which must be already allocated) - return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize) - or 0 if compression fails -*/ - - -/*-************************************ -* Advanced Functions -**************************************/ -#define LIZ_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ -#define LIZ_BLOCK_SIZE (1<<17) -#define LIZ_BLOCK64K_SIZE (1<<16) -#define LIZ_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LIZ_MAX_INPUT_SIZE ? 0 : (isize) + 1 + 1 + ((isize/LIZ_BLOCK_SIZE)+1)*4) - - -/*! -LIZ_compressBound() : - Provides the maximum size that LZ5 compression may output in a "worst case" scenario (input data not compressible) - This function is primarily useful for memory allocation purposes (destination buffer size). - Macro LIZ_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). - Note that LIZ_compress() compress faster when dest buffer size is >= LIZ_compressBound(srcSize) - inputSize : max supported value is LIZ_MAX_INPUT_SIZE - return : maximum output size in a "worst case" scenario - or 0, if input size is too large ( > LIZ_MAX_INPUT_SIZE) -*/ -LZ5LIB_API int LIZ_compressBound(int inputSize); - - -/*! -LIZ_compress_extState() : - Same compression function, just using an externally allocated memory space to store compression state. - Use LIZ_sizeofState() to know how much memory must be allocated, - and allocate it on 8-bytes boundaries (using malloc() typically). - Then, provide it as 'void* state' to compression function. -*/ -LZ5LIB_API int LIZ_sizeofState(int compressionLevel); - -LZ5LIB_API int LIZ_compress_extState(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); - - - -/*-********************************************* -* Streaming Compression Functions -***********************************************/ - -/*! LIZ_createStream() will allocate and initialize an `LIZ_stream_t` structure. - * LIZ_freeStream() releases its memory. - * In the context of a DLL (liblz5), please use these methods rather than the static struct. - * They are more future proof, in case of a change of `LIZ_stream_t` size. - */ -LZ5LIB_API LIZ_stream_t* LIZ_createStream(int compressionLevel); -LZ5LIB_API int LIZ_freeStream (LIZ_stream_t* streamPtr); - - -/*! LIZ_resetStream() : - * Use this function to reset/reuse an allocated `LIZ_stream_t` structure - */ -LZ5LIB_API LIZ_stream_t* LIZ_resetStream (LIZ_stream_t* streamPtr, int compressionLevel); - - -/*! LIZ_loadDict() : - * Use this function to load a static dictionary into LIZ_stream. - * Any previous data will be forgotten, only 'dictionary' will remain in memory. - * Loading a size of 0 is allowed. - * Return : dictionary size, in bytes (necessarily <= LIZ_DICT_SIZE) - */ -LZ5LIB_API int LIZ_loadDict (LIZ_stream_t* streamPtr, const char* dictionary, int dictSize); - - -/*! LIZ_compress_continue() : - * Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio. - * Important : Previous data blocks are assumed to still be present and unmodified ! - * 'dst' buffer must be already allocated. - * If maxDstSize >= LIZ_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. - * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero. - */ -LZ5LIB_API int LIZ_compress_continue (LIZ_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize); - - -/*! LIZ_saveDict() : - * If previously compressed data block is not guaranteed to remain available at its memory location, - * save it into a safer place (char* safeBuffer). - * Note : you don't need to call LIZ_loadDict() afterwards, - * dictionary is immediately usable, you can therefore call LIZ_compress_continue(). - * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. - */ -LZ5LIB_API int LIZ_saveDict (LIZ_stream_t* streamPtr, char* safeBuffer, int dictSize); - - - - - -#if defined (__cplusplus) -} -#endif - -#endif /* LIZ_H_2983827168210 */ diff --git a/C/lizard/liz_entropy_common.c b/C/lizard/liz_entropy_common.c deleted file mode 100644 index 61a1ad73..00000000 --- a/C/lizard/liz_entropy_common.c +++ /dev/null @@ -1,222 +0,0 @@ -/* - Common functions of New Generation Entropy library - Copyright (C) 2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -*************************************************************************** */ - -/* ************************************* -* Dependencies -***************************************/ -#include "mem.h" -#include "error_private.h" /* ERR_*, ERROR */ -#define LIZFSE_STATIC_LINKING_ONLY /* LIZFSE_MIN_TABLELOG */ -#include "fse.h" -#define LIZHUF_STATIC_LINKING_ONLY /* LIZHUF_TABLELOG_ABSOLUTEMAX */ -#include "huf.h" - - -/*-**************************************** -* FSE Error Management -******************************************/ -unsigned LIZFSE_isError(size_t code) { return ERR_isError(code); } - -const char* LIZFSE_getErrorName(size_t code) { return ERR_getErrorName(code); } - - -/* ************************************************************** -* HUF Error Management -****************************************************************/ -unsigned LIZHUF_isError(size_t code) { return ERR_isError(code); } - -const char* LIZHUF_getErrorName(size_t code) { return ERR_getErrorName(code); } - - -/*-************************************************************** -* FSE NCount encoding-decoding -****************************************************************/ -static short LIZFSE_abs(short a) { return (short)(a<0 ? -a : a); } - -size_t LIZFSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, - const void* headerBuffer, size_t hbSize) -{ - const BYTE* const istart = (const BYTE*) headerBuffer; - const BYTE* const iend = istart + hbSize; - const BYTE* ip = istart; - int nbBits; - int remaining; - int threshold; - U32 bitStream; - int bitCount; - unsigned charnum = 0; - int previous0 = 0; - - if (hbSize < 4) return ERROR(srcSize_wrong); - bitStream = MEM_readLE32(ip); - nbBits = (bitStream & 0xF) + LIZFSE_MIN_TABLELOG; /* extract tableLog */ - if (nbBits > LIZFSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); - bitStream >>= 4; - bitCount = 4; - *tableLogPtr = nbBits; - remaining = (1<1) & (charnum<=*maxSVPtr)) { - if (previous0) { - unsigned n0 = charnum; - while ((bitStream & 0xFFFF) == 0xFFFF) { - n0 += 24; - if (ip < iend-5) { - ip += 2; - bitStream = MEM_readLE32(ip) >> bitCount; - } else { - bitStream >>= 16; - bitCount += 16; - } } - while ((bitStream & 3) == 3) { - n0 += 3; - bitStream >>= 2; - bitCount += 2; - } - n0 += bitStream & 3; - bitCount += 2; - if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall); - while (charnum < n0) normalizedCounter[charnum++] = 0; - if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { - ip += bitCount>>3; - bitCount &= 7; - bitStream = MEM_readLE32(ip) >> bitCount; - } else { - bitStream >>= 2; - } } - { short const max = (short)((2*threshold-1)-remaining); - short count; - - if ((bitStream & (threshold-1)) < (U32)max) { - count = (short)(bitStream & (threshold-1)); - bitCount += nbBits-1; - } else { - count = (short)(bitStream & (2*threshold-1)); - if (count >= threshold) count -= max; - bitCount += nbBits; - } - - count--; /* extra accuracy */ - remaining -= LIZFSE_abs(count); - normalizedCounter[charnum++] = count; - previous0 = !count; - while (remaining < threshold) { - nbBits--; - threshold >>= 1; - } - - if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { - ip += bitCount>>3; - bitCount &= 7; - } else { - bitCount -= (int)(8 * (iend - 4 - ip)); - ip = iend - 4; - } - bitStream = MEM_readLE32(ip) >> (bitCount & 31); - } } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */ - if (remaining != 1) return ERROR(corruption_detected); - if (bitCount > 32) return ERROR(corruption_detected); - *maxSVPtr = charnum-1; - - ip += (bitCount+7)>>3; - return ip-istart; -} - - -/*! LIZHUF_readStats() : - Read compact Huffman tree, saved by LIZHUF_writeCTable(). - `huffWeight` is destination buffer. - @return : size read from `src` , or an error Code . - Note : Needed by LIZHUF_readCTable() and LIZHUF_readDTableX?() . -*/ -size_t LIZHUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, - U32* nbSymbolsPtr, U32* tableLogPtr, - const void* src, size_t srcSize) -{ - U32 weightTotal; - const BYTE* ip = (const BYTE*) src; - size_t iSize = ip[0]; - size_t oSize; - - /* memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */ - - if (iSize >= 128) { /* special header */ - oSize = iSize - 127; - iSize = ((oSize+1)/2); - if (iSize+1 > srcSize) return ERROR(srcSize_wrong); - if (oSize >= hwSize) return ERROR(corruption_detected); - ip += 1; - { U32 n; - for (n=0; n> 4; - huffWeight[n+1] = ip[n/2] & 15; - } } } - else { /* header compressed with FSE (normal case) */ - if (iSize+1 > srcSize) return ERROR(srcSize_wrong); - oSize = LIZFSE_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */ - if (LIZFSE_isError(oSize)) return oSize; - } - - /* collect weight stats */ - memset(rankStats, 0, (LIZHUF_TABLELOG_ABSOLUTEMAX + 1) * sizeof(U32)); - weightTotal = 0; - { U32 n; for (n=0; n= LIZHUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected); - rankStats[huffWeight[n]]++; - weightTotal += (1 << huffWeight[n]) >> 1; - } } - - /* get last non-null symbol weight (implied, total must be 2^n) */ - { U32 const tableLog = BIT_highbit32(weightTotal) + 1; - if (tableLog > LIZHUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected); - *tableLogPtr = tableLog; - /* determine last weight */ - { U32 const total = 1 << tableLog; - U32 const rest = total - weightTotal; - U32 const verif = 1 << BIT_highbit32(rest); - U32 const lastWeight = BIT_highbit32(rest) + 1; - if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ - huffWeight[oSize] = (BYTE)lastWeight; - rankStats[lastWeight]++; - } } - - /* check tree construction validity */ - if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ - - /* results */ - *nbSymbolsPtr = (U32)(oSize+1); - return iSize+1; -} diff --git a/C/lizard/liz_fse_compress.c b/C/lizard/liz_fse_compress.c deleted file mode 100644 index 91216464..00000000 --- a/C/lizard/liz_fse_compress.c +++ /dev/null @@ -1,810 +0,0 @@ -/* ****************************************************************** - FSE : Finite State Entropy encoder - Copyright (C) 2013-2015, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -****************************************************************** */ - -/* ************************************************************** -* Compiler specifics -****************************************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline -# include /* For Visual 2005 */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ -#else -# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -# else -# define FORCE_INLINE static -# endif /* __STDC_VERSION__ */ -#endif - - -/* ************************************************************** -* Includes -****************************************************************/ -#include /* malloc, free, qsort */ -#include /* memcpy, memset */ -#include /* printf (debug) */ -#include "bitstream.h" -#define LIZFSE_STATIC_LINKING_ONLY -#include "fse.h" - - -/* ************************************************************** -* Error Management -****************************************************************/ -#define LIZFSE_STATIC_ASSERT(c) { enum { LIZFSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ - - -/* ************************************************************** -* Complex types -****************************************************************/ -typedef U32 CTable_max_t[LIZFSE_CTABLE_SIZE_U32(LIZFSE_MAX_TABLELOG, LIZFSE_MAX_SYMBOL_VALUE)]; - - -/* ************************************************************** -* Templates -****************************************************************/ -/* - designed to be included - for type-specific functions (template emulation in C) - Objective is to write these functions only once, for improved maintenance -*/ - -/* safety checks */ -#ifndef LIZFSE_FUNCTION_EXTENSION -# error "LIZFSE_FUNCTION_EXTENSION must be defined" -#endif -#ifndef LIZFSE_FUNCTION_TYPE -# error "LIZFSE_FUNCTION_TYPE must be defined" -#endif - -/* Function names */ -#define LIZFSE_CAT(X,Y) X##Y -#define LIZFSE_FUNCTION_NAME(X,Y) LIZFSE_CAT(X,Y) -#define LIZFSE_TYPE_NAME(X,Y) LIZFSE_CAT(X,Y) - - -/* Function templates */ -size_t LIZFSE_buildCTable(LIZFSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) -{ - U32 const tableSize = 1 << tableLog; - U32 const tableMask = tableSize - 1; - void* const ptr = ct; - U16* const tableU16 = ( (U16*) ptr) + 2; - void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableLog ? tableSize>>1 : 1) ; - LIZFSE_symbolCompressionTransform* const symbolTT = (LIZFSE_symbolCompressionTransform*) (FSCT); - U32 const step = LIZFSE_TABLESTEP(tableSize); - U32 cumul[LIZFSE_MAX_SYMBOL_VALUE+2]; - - LIZFSE_FUNCTION_TYPE tableSymbol[LIZFSE_MAX_TABLESIZE]; /* memset() is not necessary, even if static analyzer complain about it */ - U32 highThreshold = tableSize-1; - - /* CTable header */ - tableU16[-2] = (U16) tableLog; - tableU16[-1] = (U16) maxSymbolValue; - - /* For explanations on how to distribute symbol values over the table : - * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ - - /* symbol start positions */ - { U32 u; - cumul[0] = 0; - for (u=1; u<=maxSymbolValue+1; u++) { - if (normalizedCounter[u-1]==-1) { /* Low proba symbol */ - cumul[u] = cumul[u-1] + 1; - tableSymbol[highThreshold--] = (LIZFSE_FUNCTION_TYPE)(u-1); - } else { - cumul[u] = cumul[u-1] + normalizedCounter[u-1]; - } } - cumul[maxSymbolValue+1] = tableSize+1; - } - - /* Spread symbols */ - { U32 position = 0; - U32 symbol; - for (symbol=0; symbol<=maxSymbolValue; symbol++) { - int nbOccurences; - for (nbOccurences=0; nbOccurences highThreshold) position = (position + step) & tableMask; /* Low proba area */ - } } - - if (position!=0) return ERROR(GENERIC); /* Must have gone through all positions */ - } - - /* Build table */ - { U32 u; for (u=0; u> 3) + 3; - return maxSymbolValue ? maxHeaderSize : LIZFSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ -} - -static short LIZFSE_abs(short a) { return (short)(a<0 ? -a : a); } - -static size_t LIZFSE_writeNCount_generic (void* header, size_t headerBufferSize, - const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, - unsigned writeIsSafe) -{ - BYTE* const ostart = (BYTE*) header; - BYTE* out = ostart; - BYTE* const oend = ostart + headerBufferSize; - int nbBits; - const int tableSize = 1 << tableLog; - int remaining; - int threshold; - U32 bitStream; - int bitCount; - unsigned charnum = 0; - int previous0 = 0; - - bitStream = 0; - bitCount = 0; - /* Table Size */ - bitStream += (tableLog-LIZFSE_MIN_TABLELOG) << bitCount; - bitCount += 4; - - /* Init */ - remaining = tableSize+1; /* +1 for extra accuracy */ - threshold = tableSize; - nbBits = tableLog+1; - - while (remaining>1) { /* stops at 1 */ - if (previous0) { - unsigned start = charnum; - while (!normalizedCounter[charnum]) charnum++; - while (charnum >= start+24) { - start+=24; - bitStream += 0xFFFFU << bitCount; - if ((!writeIsSafe) && (out > oend-2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE) bitStream; - out[1] = (BYTE)(bitStream>>8); - out+=2; - bitStream>>=16; - } - while (charnum >= start+3) { - start+=3; - bitStream += 3 << bitCount; - bitCount += 2; - } - bitStream += (charnum-start) << bitCount; - bitCount += 2; - if (bitCount>16) { - if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE)bitStream; - out[1] = (BYTE)(bitStream>>8); - out += 2; - bitStream >>= 16; - bitCount -= 16; - } } - { short count = normalizedCounter[charnum++]; - const short max = (short)((2*threshold-1)-remaining); - remaining -= LIZFSE_abs(count); - if (remaining<1) return ERROR(GENERIC); - count++; /* +1 for extra accuracy */ - if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ - bitStream += count << bitCount; - bitCount += nbBits; - bitCount -= (count>=1; - } - if (bitCount>16) { - if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE)bitStream; - out[1] = (BYTE)(bitStream>>8); - out += 2; - bitStream >>= 16; - bitCount -= 16; - } } - - /* flush remaining bitStream */ - if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE)bitStream; - out[1] = (BYTE)(bitStream>>8); - out+= (bitCount+7) /8; - - if (charnum > maxSymbolValue + 1) return ERROR(GENERIC); - - return (out-ostart); -} - - -size_t LIZFSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) -{ - if (tableLog > LIZFSE_MAX_TABLELOG) return ERROR(GENERIC); /* Unsupported */ - if (tableLog < LIZFSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */ - - if (bufferSize < LIZFSE_NCountWriteBound(maxSymbolValue, tableLog)) - return LIZFSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); - - return LIZFSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1); -} - - - -/*-************************************************************** -* Counting histogram -****************************************************************/ -/*! LIZFSE_count_simple - This function just counts byte values within `src`, - and store the histogram into table `count`. - This function is unsafe : it doesn't check that all values within `src` can fit into `count`. - For this reason, prefer using a table `count` with 256 elements. - @return : count of most numerous element -*/ -static size_t LIZFSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, - const void* src, size_t srcSize) -{ - const BYTE* ip = (const BYTE*)src; - const BYTE* const end = ip + srcSize; - unsigned maxSymbolValue = *maxSymbolValuePtr; - unsigned max=0; - - - memset(count, 0, (maxSymbolValue+1)*sizeof(*count)); - if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; } - - while (ip max) max = count[s]; } - - return (size_t)max; -} - - -static size_t LIZFSE_count_parallel(unsigned* count, unsigned* maxSymbolValuePtr, - const void* source, size_t sourceSize, - unsigned checkMax) -{ - const BYTE* ip = (const BYTE*)source; - const BYTE* const iend = ip+sourceSize; - unsigned maxSymbolValue = *maxSymbolValuePtr; - unsigned max=0; - - - U32 Counting1[256] = { 0 }; - U32 Counting2[256] = { 0 }; - U32 Counting3[256] = { 0 }; - U32 Counting4[256] = { 0 }; - - /* safety checks */ - if (!sourceSize) { - memset(count, 0, maxSymbolValue + 1); - *maxSymbolValuePtr = 0; - return 0; - } - if (!maxSymbolValue) maxSymbolValue = 255; /* 0 == default */ - - /* by stripes of 16 bytes */ - { U32 cached = MEM_read32(ip); ip += 4; - while (ip < iend-15) { - U32 c = cached; cached = MEM_read32(ip); ip += 4; - Counting1[(BYTE) c ]++; - Counting2[(BYTE)(c>>8) ]++; - Counting3[(BYTE)(c>>16)]++; - Counting4[ c>>24 ]++; - c = cached; cached = MEM_read32(ip); ip += 4; - Counting1[(BYTE) c ]++; - Counting2[(BYTE)(c>>8) ]++; - Counting3[(BYTE)(c>>16)]++; - Counting4[ c>>24 ]++; - c = cached; cached = MEM_read32(ip); ip += 4; - Counting1[(BYTE) c ]++; - Counting2[(BYTE)(c>>8) ]++; - Counting3[(BYTE)(c>>16)]++; - Counting4[ c>>24 ]++; - c = cached; cached = MEM_read32(ip); ip += 4; - Counting1[(BYTE) c ]++; - Counting2[(BYTE)(c>>8) ]++; - Counting3[(BYTE)(c>>16)]++; - Counting4[ c>>24 ]++; - } - ip-=4; - } - - /* finish last symbols */ - while (ipmaxSymbolValue; s--) { - Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s]; - if (Counting1[s]) return ERROR(maxSymbolValue_tooSmall); - } } - - { U32 s; for (s=0; s<=maxSymbolValue; s++) { - count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s]; - if (count[s] > max) max = count[s]; - }} - - while (!count[maxSymbolValue]) maxSymbolValue--; - *maxSymbolValuePtr = maxSymbolValue; - return (size_t)max; -} - -/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */ -size_t LIZFSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, - const void* source, size_t sourceSize) -{ - if (sourceSize < 1500) return LIZFSE_count_simple(count, maxSymbolValuePtr, source, sourceSize); - return LIZFSE_count_parallel(count, maxSymbolValuePtr, source, sourceSize, 0); -} - -size_t LIZFSE_count(unsigned* count, unsigned* maxSymbolValuePtr, - const void* source, size_t sourceSize) -{ - if (*maxSymbolValuePtr <255) - return LIZFSE_count_parallel(count, maxSymbolValuePtr, source, sourceSize, 1); - *maxSymbolValuePtr = 255; - return LIZFSE_countFast(count, maxSymbolValuePtr, source, sourceSize); -} - - - -/*-************************************************************** -* FSE Compression Code -****************************************************************/ -/*! LIZFSE_sizeof_CTable() : - LIZFSE_CTable is a variable size structure which contains : - `U16 tableLog;` - `U16 maxSymbolValue;` - `U16 nextStateNumber[1 << tableLog];` // This size is variable - `LIZFSE_symbolCompressionTransform symbolTT[maxSymbolValue+1];` // This size is variable -Allocation is manual (C standard does not support variable-size structures). -*/ - -size_t LIZFSE_sizeof_CTable (unsigned maxSymbolValue, unsigned tableLog) -{ - size_t size; - LIZFSE_STATIC_ASSERT((size_t)LIZFSE_CTABLE_SIZE_U32(LIZFSE_MAX_TABLELOG, LIZFSE_MAX_SYMBOL_VALUE)*4 >= sizeof(CTable_max_t)); /* A compilation error here means LIZFSE_CTABLE_SIZE_U32 is not large enough */ - if (tableLog > LIZFSE_MAX_TABLELOG) return ERROR(GENERIC); - size = LIZFSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32); - return size; -} - -LIZFSE_CTable* LIZFSE_createCTable (unsigned maxSymbolValue, unsigned tableLog) -{ - size_t size; - if (tableLog > LIZFSE_TABLELOG_ABSOLUTE_MAX) tableLog = LIZFSE_TABLELOG_ABSOLUTE_MAX; - size = LIZFSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32); - return (LIZFSE_CTable*)malloc(size); -} - -void LIZFSE_freeCTable (LIZFSE_CTable* ct) { free(ct); } - -/* provides the minimum logSize to safely represent a distribution */ -static unsigned LIZFSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) -{ - U32 minBitsSrc = BIT_highbit32((U32)(srcSize - 1)) + 1; - U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2; - U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; - return minBits; -} - -unsigned LIZFSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus) -{ - U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus; - U32 tableLog = maxTableLog; - U32 minBits = LIZFSE_minTableLog(srcSize, maxSymbolValue); - if (tableLog==0) tableLog = LIZFSE_DEFAULT_TABLELOG; - if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */ - if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */ - if (tableLog < LIZFSE_MIN_TABLELOG) tableLog = LIZFSE_MIN_TABLELOG; - if (tableLog > LIZFSE_MAX_TABLELOG) tableLog = LIZFSE_MAX_TABLELOG; - return tableLog; -} - -unsigned LIZFSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) -{ - return LIZFSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2); -} - - -/* Secondary normalization method. - To be used when primary method fails. */ - -static size_t LIZFSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue) -{ - U32 s; - U32 distributed = 0; - U32 ToDistribute; - - /* Init */ - U32 lowThreshold = (U32)(total >> tableLog); - U32 lowOne = (U32)((total * 3) >> (tableLog + 1)); - - for (s=0; s<=maxSymbolValue; s++) { - if (count[s] == 0) { - norm[s]=0; - continue; - } - if (count[s] <= lowThreshold) { - norm[s] = -1; - distributed++; - total -= count[s]; - continue; - } - if (count[s] <= lowOne) { - norm[s] = 1; - distributed++; - total -= count[s]; - continue; - } - norm[s]=-2; - } - ToDistribute = (1 << tableLog) - distributed; - - if ((total / ToDistribute) > lowOne) { - /* risk of rounding to zero */ - lowOne = (U32)((total * 3) / (ToDistribute * 2)); - for (s=0; s<=maxSymbolValue; s++) { - if ((norm[s] == -2) && (count[s] <= lowOne)) { - norm[s] = 1; - distributed++; - total -= count[s]; - continue; - } } - ToDistribute = (1 << tableLog) - distributed; - } - - if (distributed == maxSymbolValue+1) { - /* all values are pretty poor; - probably incompressible data (should have already been detected); - find max, then give all remaining points to max */ - U32 maxV = 0, maxC = 0; - for (s=0; s<=maxSymbolValue; s++) - if (count[s] > maxC) maxV=s, maxC=count[s]; - norm[maxV] += (short)ToDistribute; - return 0; - } - - { - U64 const vStepLog = 62 - tableLog; - U64 const mid = (1ULL << (vStepLog-1)) - 1; - U64 const rStep = ((((U64)1<> vStepLog); - U32 sEnd = (U32)(end >> vStepLog); - U32 weight = sEnd - sStart; - if (weight < 1) - return ERROR(GENERIC); - norm[s] = (short)weight; - tmpTotal = end; - } } } - - return 0; -} - - -size_t LIZFSE_normalizeCount (short* normalizedCounter, unsigned tableLog, - const unsigned* count, size_t total, - unsigned maxSymbolValue) -{ - /* Sanity checks */ - if (tableLog==0) tableLog = LIZFSE_DEFAULT_TABLELOG; - if (tableLog < LIZFSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported size */ - if (tableLog > LIZFSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported size */ - if (tableLog < LIZFSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */ - - { U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 }; - - U64 const scale = 62 - tableLog; - U64 const step = ((U64)1<<62) / total; /* <== here, one division ! */ - U64 const vStep = 1ULL<<(scale-20); - int stillToDistribute = 1<> tableLog); - - for (s=0; s<=maxSymbolValue; s++) { - if (count[s] == total) return 0; /* rle special case */ - if (count[s] == 0) { normalizedCounter[s]=0; continue; } - if (count[s] <= lowThreshold) { - normalizedCounter[s] = -1; - stillToDistribute--; - } else { - short proba = (short)((count[s]*step) >> scale); - if (proba<8) { - U64 restToBeat = vStep * rtbTable[proba]; - proba += (count[s]*step) - ((U64)proba< restToBeat; - } - if (proba > largestP) largestP=proba, largest=s; - normalizedCounter[s] = proba; - stillToDistribute -= proba; - } } - if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) { - /* corner case, need another normalization method */ - size_t errorCode = LIZFSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue); - if (LIZFSE_isError(errorCode)) return errorCode; - } - else normalizedCounter[largest] += (short)stillToDistribute; - } - -#if 0 - { /* Print Table (debug) */ - U32 s; - U32 nTotal = 0; - for (s=0; s<=maxSymbolValue; s++) - printf("%3i: %4i \n", s, normalizedCounter[s]); - for (s=0; s<=maxSymbolValue; s++) - nTotal += abs(normalizedCounter[s]); - if (nTotal != (1U<>1); /* assumption : tableLog >= 1 */ - LIZFSE_symbolCompressionTransform* const symbolTT = (LIZFSE_symbolCompressionTransform*) (FSCT); - unsigned s; - - /* Sanity checks */ - if (nbBits < 1) return ERROR(GENERIC); /* min size */ - - /* header */ - tableU16[-2] = (U16) nbBits; - tableU16[-1] = (U16) maxSymbolValue; - - /* Build table */ - for (s=0; s LIZFSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) { /* test bit 2 */ - LIZFSE_encodeSymbol(&bitC, &CState2, *--ip); - LIZFSE_encodeSymbol(&bitC, &CState1, *--ip); - LIZFSE_FLUSHBITS(&bitC); - } - - /* 2 or 4 encoding per loop */ - for ( ; ip>istart ; ) { - - LIZFSE_encodeSymbol(&bitC, &CState2, *--ip); - - if (sizeof(bitC.bitContainer)*8 < LIZFSE_MAX_TABLELOG*2+7 ) /* this test must be static */ - LIZFSE_FLUSHBITS(&bitC); - - LIZFSE_encodeSymbol(&bitC, &CState1, *--ip); - - if (sizeof(bitC.bitContainer)*8 > LIZFSE_MAX_TABLELOG*4+7 ) { /* this test must be static */ - LIZFSE_encodeSymbol(&bitC, &CState2, *--ip); - LIZFSE_encodeSymbol(&bitC, &CState1, *--ip); - } - - LIZFSE_FLUSHBITS(&bitC); - } - - LIZFSE_flushCState(&bitC, &CState2); - LIZFSE_flushCState(&bitC, &CState1); - return BIT_closeCStream(&bitC); -} - -size_t LIZFSE_compress_usingCTable (void* dst, size_t dstSize, - const void* src, size_t srcSize, - const LIZFSE_CTable* ct) -{ - const unsigned fast = (dstSize >= LIZFSE_BLOCKBOUND(srcSize)); - - if (fast) - return LIZFSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1); - else - return LIZFSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0); -} - - -size_t LIZFSE_compressBound(size_t size) { return LIZFSE_COMPRESSBOUND(size); } - -size_t LIZFSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog) -{ - const BYTE* const istart = (const BYTE*) src; - const BYTE* ip = istart; - - BYTE* const ostart = (BYTE*) dst; - BYTE* op = ostart; - BYTE* const oend = ostart + dstSize; - - U32 count[LIZFSE_MAX_SYMBOL_VALUE+1]; - S16 norm[LIZFSE_MAX_SYMBOL_VALUE+1]; - CTable_max_t ct; - size_t errorCode; - - /* init conditions */ - if (srcSize <= 1) return 0; /* Uncompressible */ - if (!maxSymbolValue) maxSymbolValue = LIZFSE_MAX_SYMBOL_VALUE; - if (!tableLog) tableLog = LIZFSE_DEFAULT_TABLELOG; - - /* Scan input and build symbol stats */ - errorCode = LIZFSE_count (count, &maxSymbolValue, ip, srcSize); - if (LIZFSE_isError(errorCode)) return errorCode; - if (errorCode == srcSize) return 1; - if (errorCode == 1) return 0; /* each symbol only present once */ - if (errorCode < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */ - - tableLog = LIZFSE_optimalTableLog(tableLog, srcSize, maxSymbolValue); - errorCode = LIZFSE_normalizeCount (norm, tableLog, count, srcSize, maxSymbolValue); - if (LIZFSE_isError(errorCode)) return errorCode; - - /* Write table description header */ - errorCode = LIZFSE_writeNCount (op, oend-op, norm, maxSymbolValue, tableLog); - if (LIZFSE_isError(errorCode)) return errorCode; - op += errorCode; - - /* Compress */ - errorCode = LIZFSE_buildCTable (ct, norm, maxSymbolValue, tableLog); - if (LIZFSE_isError(errorCode)) return errorCode; - errorCode = LIZFSE_compress_usingCTable(op, oend - op, ip, srcSize, ct); - if (errorCode == 0) return 0; /* not enough space for compressed data */ - op += errorCode; - - /* check compressibility */ - if ( (size_t)(op-ostart) >= srcSize-1 ) - return 0; - - return op-ostart; -} - -size_t LIZFSE_compress (void* dst, size_t dstSize, const void* src, size_t srcSize) -{ - return LIZFSE_compress2(dst, dstSize, src, (U32)srcSize, LIZFSE_MAX_SYMBOL_VALUE, LIZFSE_DEFAULT_TABLELOG); -} - - -#endif /* LIZFSE_COMMONDEFS_ONLY */ diff --git a/C/lizard/liz_fse_decompress.c b/C/lizard/liz_fse_decompress.c deleted file mode 100644 index 4de9faa6..00000000 --- a/C/lizard/liz_fse_decompress.c +++ /dev/null @@ -1,329 +0,0 @@ -/* ****************************************************************** - FSE : Finite State Entropy decoder - Copyright (C) 2013-2015, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -****************************************************************** */ - - -/* ************************************************************** -* Compiler specifics -****************************************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline -# include /* For Visual 2005 */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ -#else -# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -# else -# define FORCE_INLINE static -# endif /* __STDC_VERSION__ */ -#endif - - -/* ************************************************************** -* Includes -****************************************************************/ -#include /* malloc, free, qsort */ -#include /* memcpy, memset */ -#include /* printf (debug) */ -#include "bitstream.h" -#define LIZFSE_STATIC_LINKING_ONLY -#include "fse.h" - - -/* ************************************************************** -* Error Management -****************************************************************/ -#define LIZFSE_isError ERR_isError -#define LIZFSE_STATIC_ASSERT(c) { enum { LIZFSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ - -/* check and forward error code */ -#define CHECK_F(f) { size_t const e = f; if (LIZFSE_isError(e)) return e; } - - -/* ************************************************************** -* Complex types -****************************************************************/ -typedef U32 DTable_max_t[LIZFSE_DTABLE_SIZE_U32(LIZFSE_MAX_TABLELOG)]; - - -/* ************************************************************** -* Templates -****************************************************************/ -/* - designed to be included - for type-specific functions (template emulation in C) - Objective is to write these functions only once, for improved maintenance -*/ - -/* safety checks */ -#ifndef LIZFSE_FUNCTION_EXTENSION -# error "LIZFSE_FUNCTION_EXTENSION must be defined" -#endif -#ifndef LIZFSE_FUNCTION_TYPE -# error "LIZFSE_FUNCTION_TYPE must be defined" -#endif - -/* Function names */ -#define LIZFSE_CAT(X,Y) X##Y -#define LIZFSE_FUNCTION_NAME(X,Y) LIZFSE_CAT(X,Y) -#define LIZFSE_TYPE_NAME(X,Y) LIZFSE_CAT(X,Y) - - -/* Function templates */ -LIZFSE_DTable* LIZFSE_createDTable (unsigned tableLog) -{ - if (tableLog > LIZFSE_TABLELOG_ABSOLUTE_MAX) tableLog = LIZFSE_TABLELOG_ABSOLUTE_MAX; - return (LIZFSE_DTable*)malloc( LIZFSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); -} - -void LIZFSE_freeDTable (LIZFSE_DTable* dt) -{ - free(dt); -} - -size_t LIZFSE_buildDTable(LIZFSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) -{ - void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ - LIZFSE_DECODE_TYPE* const tableDecode = (LIZFSE_DECODE_TYPE*) (tdPtr); - U16 symbolNext[LIZFSE_MAX_SYMBOL_VALUE+1]; - - U32 const maxSV1 = maxSymbolValue + 1; - U32 const tableSize = 1 << tableLog; - U32 highThreshold = tableSize-1; - - /* Sanity Checks */ - if (maxSymbolValue > LIZFSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); - if (tableLog > LIZFSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); - - /* Init, lay down lowprob symbols */ - { LIZFSE_DTableHeader DTableH; - DTableH.tableLog = (U16)tableLog; - DTableH.fastMode = 1; - { S16 const largeLimit= (S16)(1 << (tableLog-1)); - U32 s; - for (s=0; s= largeLimit) DTableH.fastMode=0; - symbolNext[s] = normalizedCounter[s]; - } } } - memcpy(dt, &DTableH, sizeof(DTableH)); - } - - /* Spread symbols */ - { U32 const tableMask = tableSize-1; - U32 const step = LIZFSE_TABLESTEP(tableSize); - U32 s, position = 0; - for (s=0; s highThreshold) position = (position + step) & tableMask; /* lowprob area */ - } } - if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ - } - - /* Build Decoding table */ - { U32 u; - for (u=0; utableLog = 0; - DTableH->fastMode = 0; - - cell->newState = 0; - cell->symbol = symbolValue; - cell->nbBits = 0; - - return 0; -} - - -size_t LIZFSE_buildDTable_raw (LIZFSE_DTable* dt, unsigned nbBits) -{ - void* ptr = dt; - LIZFSE_DTableHeader* const DTableH = (LIZFSE_DTableHeader*)ptr; - void* dPtr = dt + 1; - LIZFSE_decode_t* const dinfo = (LIZFSE_decode_t*)dPtr; - const unsigned tableSize = 1 << nbBits; - const unsigned tableMask = tableSize - 1; - const unsigned maxSV1 = tableMask+1; - unsigned s; - - /* Sanity checks */ - if (nbBits < 1) return ERROR(GENERIC); /* min size */ - - /* Build Decoding Table */ - DTableH->tableLog = (U16)nbBits; - DTableH->fastMode = 1; - for (s=0; s sizeof(bitD.bitContainer)*8) /* This test must be static */ - BIT_reloadDStream(&bitD); - - op[1] = LIZFSE_GETSYMBOL(&state2); - - if (LIZFSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ - { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } } - - op[2] = LIZFSE_GETSYMBOL(&state1); - - if (LIZFSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ - BIT_reloadDStream(&bitD); - - op[3] = LIZFSE_GETSYMBOL(&state2); - } - - /* tail */ - /* note : BIT_reloadDStream(&bitD) >= LIZFSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ - while (1) { - if (op>(omax-2)) return ERROR(dstSize_tooSmall); - *op++ = LIZFSE_GETSYMBOL(&state1); - if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { - *op++ = LIZFSE_GETSYMBOL(&state2); - break; - } - - if (op>(omax-2)) return ERROR(dstSize_tooSmall); - *op++ = LIZFSE_GETSYMBOL(&state2); - if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { - *op++ = LIZFSE_GETSYMBOL(&state1); - break; - } } - - return op-ostart; -} - - -size_t LIZFSE_decompress_usingDTable(void* dst, size_t originalSize, - const void* cSrc, size_t cSrcSize, - const LIZFSE_DTable* dt) -{ - const void* ptr = dt; - const LIZFSE_DTableHeader* DTableH = (const LIZFSE_DTableHeader*)ptr; - const U32 fastMode = DTableH->fastMode; - - /* select fast mode (static) */ - if (fastMode) return LIZFSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); - return LIZFSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); -} - - -size_t LIZFSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize) -{ - const BYTE* const istart = (const BYTE*)cSrc; - const BYTE* ip = istart; - short counting[LIZFSE_MAX_SYMBOL_VALUE+1]; - DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */ - unsigned tableLog; - unsigned maxSymbolValue = LIZFSE_MAX_SYMBOL_VALUE; - - if (cSrcSize<2) return ERROR(srcSize_wrong); /* too small input size */ - - /* normal FSE decoding mode */ - { size_t const NCountLength = LIZFSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); - if (LIZFSE_isError(NCountLength)) return NCountLength; - if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size */ - ip += NCountLength; - cSrcSize -= NCountLength; - } - - CHECK_F( LIZFSE_buildDTable (dt, counting, maxSymbolValue, tableLog) ); - - return LIZFSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt); /* always return, even if it is an error code */ -} - - - -#endif /* LIZFSE_COMMONDEFS_ONLY */ diff --git a/C/lizard/liz_huf_compress.c b/C/lizard/liz_huf_compress.c deleted file mode 100644 index 1854e5f5..00000000 --- a/C/lizard/liz_huf_compress.c +++ /dev/null @@ -1,532 +0,0 @@ -/* ****************************************************************** - Huffman encoder, part of New Generation Entropy library - Copyright (C) 2013-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -****************************************************************** */ - -/* ************************************************************** -* Compiler specifics -****************************************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#endif - - -/* ************************************************************** -* Includes -****************************************************************/ -#include /* memcpy, memset */ -#include /* printf (debug) */ -#include "bitstream.h" -#define LIZFSE_STATIC_LINKING_ONLY /* LIZFSE_optimalTableLog_internal */ -#include "fse.h" /* header compression */ -#define LIZHUF_STATIC_LINKING_ONLY -#include "huf.h" - - -/* ************************************************************** -* Error Management -****************************************************************/ -#define LIZHUF_STATIC_ASSERT(c) { enum { LIZHUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ - - -/* ************************************************************** -* Utils -****************************************************************/ -unsigned LIZHUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) -{ - return LIZFSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); -} - - -/* ******************************************************* -* HUF : Huffman block compression -*********************************************************/ -struct LIZHUF_CElt_s { - U16 val; - BYTE nbBits; -}; /* typedef'd to LIZHUF_CElt within "huf.h" */ - -typedef struct nodeElt_s { - U32 count; - U16 parent; - BYTE byte; - BYTE nbBits; -} nodeElt; - -/*! LIZHUF_writeCTable() : - `CTable` : huffman tree to save, using huf representation. - @return : size of saved CTable */ -size_t LIZHUF_writeCTable (void* dst, size_t maxDstSize, - const LIZHUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog) -{ - BYTE bitsToWeight[LIZHUF_TABLELOG_MAX + 1]; - BYTE huffWeight[LIZHUF_SYMBOLVALUE_MAX]; - BYTE* op = (BYTE*)dst; - U32 n; - - /* check conditions */ - if (maxSymbolValue > LIZHUF_SYMBOLVALUE_MAX) return ERROR(GENERIC); - - /* convert to weight */ - bitsToWeight[0] = 0; - for (n=1; n1) & (size < maxSymbolValue/2)) { /* FSE compressed */ - op[0] = (BYTE)size; - return size+1; - } - } - - /* raw values */ - if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen */ - if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */ - op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1)); - huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */ - for (n=0; n LIZHUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); - if (nbSymbols > maxSymbolValue+1) return ERROR(maxSymbolValue_tooSmall); - - /* Prepare base value per rank */ - { U32 n, nextRankStart = 0; - for (n=1; n<=tableLog; n++) { - U32 current = nextRankStart; - nextRankStart += (rankVal[n] << (n-1)); - rankVal[n] = current; - } } - - /* fill nbBits */ - { U32 n; for (n=0; n0; n--) { - valPerRank[n] = min; /* get starting value within each rank */ - min += nbPerRank[n]; - min >>= 1; - } } - /* assign value within rank, symbol order */ - { U32 n; for (n=0; n<=maxSymbolValue; n++) CTable[n].val = valPerRank[CTable[n].nbBits]++; } - } - - return readSize; -} - - -static U32 LIZHUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) -{ - const U32 largestBits = huffNode[lastNonNull].nbBits; - if (largestBits <= maxNbBits) return largestBits; /* early exit : no elt > maxNbBits */ - - /* there are several too large elements (at least >= 2) */ - { int totalCost = 0; - const U32 baseCost = 1 << (largestBits - maxNbBits); - U32 n = lastNonNull; - - while (huffNode[n].nbBits > maxNbBits) { - totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); - huffNode[n].nbBits = (BYTE)maxNbBits; - n --; - } /* n stops at huffNode[n].nbBits <= maxNbBits */ - while (huffNode[n].nbBits == maxNbBits) n--; /* n end at index of smallest symbol using < maxNbBits */ - - /* renorm totalCost */ - totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */ - - /* repay normalized cost */ - { U32 const noSymbol = 0xF0F0F0F0; - U32 rankLast[LIZHUF_TABLELOG_MAX+2]; - int pos; - - /* Get pos of last (smallest) symbol per rank */ - memset(rankLast, 0xF0, sizeof(rankLast)); - { U32 currentNbBits = maxNbBits; - for (pos=n ; pos >= 0; pos--) { - if (huffNode[pos].nbBits >= currentNbBits) continue; - currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */ - rankLast[maxNbBits-currentNbBits] = pos; - } } - - while (totalCost > 0) { - U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1; - for ( ; nBitsToDecrease > 1; nBitsToDecrease--) { - U32 highPos = rankLast[nBitsToDecrease]; - U32 lowPos = rankLast[nBitsToDecrease-1]; - if (highPos == noSymbol) continue; - if (lowPos == noSymbol) break; - { U32 const highTotal = huffNode[highPos].count; - U32 const lowTotal = 2 * huffNode[lowPos].count; - if (highTotal <= lowTotal) break; - } } - /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ - while ((nBitsToDecrease<=LIZHUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) /* LIZHUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ - nBitsToDecrease ++; - totalCost -= 1 << (nBitsToDecrease-1); - if (rankLast[nBitsToDecrease-1] == noSymbol) - rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; /* this rank is no longer empty */ - huffNode[rankLast[nBitsToDecrease]].nbBits ++; - if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */ - rankLast[nBitsToDecrease] = noSymbol; - else { - rankLast[nBitsToDecrease]--; - if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease) - rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ - } } /* while (totalCost > 0) */ - - while (totalCost < 0) { /* Sometimes, cost correction overshoot */ - if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */ - while (huffNode[n].nbBits == maxNbBits) n--; - huffNode[n+1].nbBits--; - rankLast[1] = n+1; - totalCost++; - continue; - } - huffNode[ rankLast[1] + 1 ].nbBits--; - rankLast[1]++; - totalCost ++; - } } } /* there are several too large elements (at least >= 2) */ - - return maxNbBits; -} - - -typedef struct { - U32 base; - U32 current; -} rankPos; - -static void LIZHUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue) -{ - rankPos rank[32]; - U32 n; - - memset(rank, 0, sizeof(rank)); - for (n=0; n<=maxSymbolValue; n++) { - U32 r = BIT_highbit32(count[n] + 1); - rank[r].base ++; - } - for (n=30; n>0; n--) rank[n-1].base += rank[n].base; - for (n=0; n<32; n++) rank[n].current = rank[n].base; - for (n=0; n<=maxSymbolValue; n++) { - U32 const c = count[n]; - U32 const r = BIT_highbit32(c+1) + 1; - U32 pos = rank[r].current++; - while ((pos > rank[r].base) && (c > huffNode[pos-1].count)) huffNode[pos]=huffNode[pos-1], pos--; - huffNode[pos].count = c; - huffNode[pos].byte = (BYTE)n; - } -} - - -#define STARTNODE (LIZHUF_SYMBOLVALUE_MAX+1) -size_t LIZHUF_buildCTable (LIZHUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits) -{ - nodeElt huffNode0[2*LIZHUF_SYMBOLVALUE_MAX+1 +1]; - nodeElt* huffNode = huffNode0 + 1; - U32 n, nonNullRank; - int lowS, lowN; - U16 nodeNb = STARTNODE; - U32 nodeRoot; - - /* safety checks */ - if (maxNbBits == 0) maxNbBits = LIZHUF_TABLELOG_DEFAULT; - if (maxSymbolValue > LIZHUF_SYMBOLVALUE_MAX) return ERROR(GENERIC); - memset(huffNode0, 0, sizeof(huffNode0)); - - /* sort, decreasing order */ - LIZHUF_sort(huffNode, count, maxSymbolValue); - - /* init for parents */ - nonNullRank = maxSymbolValue; - while(huffNode[nonNullRank].count == 0) nonNullRank--; - lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb; - huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count; - huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb; - nodeNb++; lowS-=2; - for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30); - huffNode0[0].count = (U32)(1U<<31); - - /* create parents */ - while (nodeNb <= nodeRoot) { - U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; - U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; - huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count; - huffNode[n1].parent = huffNode[n2].parent = nodeNb; - nodeNb++; - } - - /* distribute weights (unlimited tree height) */ - huffNode[nodeRoot].nbBits = 0; - for (n=nodeRoot-1; n>=STARTNODE; n--) - huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; - for (n=0; n<=nonNullRank; n++) - huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; - - /* enforce maxTableLog */ - maxNbBits = LIZHUF_setMaxHeight(huffNode, nonNullRank, maxNbBits); - - /* fill result into tree (val, nbBits) */ - { U16 nbPerRank[LIZHUF_TABLELOG_MAX+1] = {0}; - U16 valPerRank[LIZHUF_TABLELOG_MAX+1] = {0}; - if (maxNbBits > LIZHUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */ - for (n=0; n<=nonNullRank; n++) - nbPerRank[huffNode[n].nbBits]++; - /* determine stating value per rank */ - { U16 min = 0; - for (n=maxNbBits; n>0; n--) { - valPerRank[n] = min; /* get starting value within each rank */ - min += nbPerRank[n]; - min >>= 1; - } } - for (n=0; n<=maxSymbolValue; n++) - tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */ - for (n=0; n<=maxSymbolValue; n++) - tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */ - } - - return maxNbBits; -} - -static void LIZHUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const LIZHUF_CElt* CTable) -{ - BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits); -} - -size_t LIZHUF_compressBound(size_t size) { return LIZHUF_COMPRESSBOUND(size); } - -#define LIZHUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s)) - -#define LIZHUF_FLUSHBITS_1(stream) \ - if (sizeof((stream)->bitContainer)*8 < LIZHUF_TABLELOG_MAX*2+7) LIZHUF_FLUSHBITS(stream) - -#define LIZHUF_FLUSHBITS_2(stream) \ - if (sizeof((stream)->bitContainer)*8 < LIZHUF_TABLELOG_MAX*4+7) LIZHUF_FLUSHBITS(stream) - -size_t LIZHUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const LIZHUF_CElt* CTable) -{ - const BYTE* ip = (const BYTE*) src; - BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + dstSize; - BYTE* op = ostart; - size_t n; - const unsigned fast = (dstSize >= LIZHUF_BLOCKBOUND(srcSize)); - BIT_CStream_t bitC; - - /* init */ - if (dstSize < 8) return 0; /* not enough space to compress */ - { size_t const errorCode = BIT_initCStream(&bitC, op, oend-op); - if (LIZHUF_isError(errorCode)) return 0; } - - n = srcSize & ~3; /* join to mod 4 */ - switch (srcSize & 3) - { - case 3 : LIZHUF_encodeSymbol(&bitC, ip[n+ 2], CTable); - LIZHUF_FLUSHBITS_2(&bitC); - case 2 : LIZHUF_encodeSymbol(&bitC, ip[n+ 1], CTable); - LIZHUF_FLUSHBITS_1(&bitC); - case 1 : LIZHUF_encodeSymbol(&bitC, ip[n+ 0], CTable); - LIZHUF_FLUSHBITS(&bitC); - case 0 : - default: ; - } - - for (; n>0; n-=4) { /* note : n&3==0 at this stage */ - LIZHUF_encodeSymbol(&bitC, ip[n- 1], CTable); - LIZHUF_FLUSHBITS_1(&bitC); - LIZHUF_encodeSymbol(&bitC, ip[n- 2], CTable); - LIZHUF_FLUSHBITS_2(&bitC); - LIZHUF_encodeSymbol(&bitC, ip[n- 3], CTable); - LIZHUF_FLUSHBITS_1(&bitC); - LIZHUF_encodeSymbol(&bitC, ip[n- 4], CTable); - LIZHUF_FLUSHBITS(&bitC); - } - - return BIT_closeCStream(&bitC); -} - - -size_t LIZHUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const LIZHUF_CElt* CTable) -{ - size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */ - const BYTE* ip = (const BYTE*) src; - const BYTE* const iend = ip + srcSize; - BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - BYTE* op = ostart; - - if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */ - if (srcSize < 12) return 0; /* no saving possible : too small input */ - op += 6; /* jumpTable */ - - { size_t const cSize = LIZHUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable); - if (LIZHUF_isError(cSize)) return cSize; - if (cSize==0) return 0; - MEM_writeLE16(ostart, (U16)cSize); - op += cSize; - } - - ip += segmentSize; - { size_t const cSize = LIZHUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable); - if (LIZHUF_isError(cSize)) return cSize; - if (cSize==0) return 0; - MEM_writeLE16(ostart+2, (U16)cSize); - op += cSize; - } - - ip += segmentSize; - { size_t const cSize = LIZHUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable); - if (LIZHUF_isError(cSize)) return cSize; - if (cSize==0) return 0; - MEM_writeLE16(ostart+4, (U16)cSize); - op += cSize; - } - - ip += segmentSize; - { size_t const cSize = LIZHUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable); - if (LIZHUF_isError(cSize)) return cSize; - if (cSize==0) return 0; - op += cSize; - } - - return op-ostart; -} - - -static size_t LIZHUF_compress_internal ( - void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog, - unsigned singleStream) -{ - BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + dstSize; - BYTE* op = ostart; - - U32 count[LIZHUF_SYMBOLVALUE_MAX+1]; - LIZHUF_CElt CTable[LIZHUF_SYMBOLVALUE_MAX+1]; - - /* checks & inits */ - if (!srcSize) return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */ - if (!dstSize) return 0; /* cannot fit within dst budget */ - if (srcSize > LIZHUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */ - if (huffLog > LIZHUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); - if (!maxSymbolValue) maxSymbolValue = LIZHUF_SYMBOLVALUE_MAX; - if (!huffLog) huffLog = LIZHUF_TABLELOG_DEFAULT; - - /* Scan input and build symbol stats */ - { size_t const largest = LIZFSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize); - if (LIZHUF_isError(largest)) return largest; - if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */ - if (largest <= (srcSize >> 7)+1) return 0; /* Fast heuristic : not compressible enough */ - } - - /* Build Huffman Tree */ - huffLog = LIZHUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); - { size_t const maxBits = LIZHUF_buildCTable (CTable, count, maxSymbolValue, huffLog); - if (LIZHUF_isError(maxBits)) return maxBits; - huffLog = (U32)maxBits; - } - - /* Write table description header */ - { size_t const hSize = LIZHUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog); - if (LIZHUF_isError(hSize)) return hSize; - if (hSize + 12 >= srcSize) return 0; /* not useful to try compression */ - op += hSize; - } - - /* Compress */ - { size_t const cSize = (singleStream) ? - LIZHUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) : /* single segment */ - LIZHUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable); - if (LIZHUF_isError(cSize)) return cSize; - if (cSize==0) return 0; /* uncompressible */ - op += cSize; - } - - /* check compressibility */ - if ((size_t)(op-ostart) >= srcSize-1) - return 0; - - return op-ostart; -} - - -size_t LIZHUF_compress1X (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog) -{ - return LIZHUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1); -} - -size_t LIZHUF_compress2 (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog) -{ - return LIZHUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0); -} - - -size_t LIZHUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize) -{ - return LIZHUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, LIZHUF_TABLELOG_DEFAULT); -} diff --git a/C/lizard/liz_huf_decompress.c b/C/lizard/liz_huf_decompress.c deleted file mode 100644 index 06a836f1..00000000 --- a/C/lizard/liz_huf_decompress.c +++ /dev/null @@ -1,883 +0,0 @@ -/* ****************************************************************** - Huffman decoder, part of New Generation Entropy library - Copyright (C) 2013-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -****************************************************************** */ - -/* ************************************************************** -* Compiler specifics -****************************************************************/ -#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -/* inline is defined */ -#elif defined(_MSC_VER) || defined(__GNUC__) -# define inline __inline -#else -# define inline /* disable inline */ -#endif - -#ifdef _MSC_VER /* Visual Studio */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#endif - - -/* ************************************************************** -* Dependencies -****************************************************************/ -#include /* memcpy, memset */ -#include "bitstream.h" /* BIT_* */ -#include "fse.h" /* header compression */ -#define LIZHUF_STATIC_LINKING_ONLY -#include "huf.h" - - -/* ************************************************************** -* Error Management -****************************************************************/ -#define LIZHUF_STATIC_ASSERT(c) { enum { LIZHUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ - - -/*-***************************/ -/* generic DTableDesc */ -/*-***************************/ - -typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc; - -static DTableDesc LIZHUF_getDTableDesc(const LIZHUF_DTable* table) -{ - DTableDesc dtd; - memcpy(&dtd, table, sizeof(dtd)); - return dtd; -} - - -/*-***************************/ -/* single-symbol decoding */ -/*-***************************/ - -typedef struct { BYTE byte; BYTE nbBits; } LIZHUF_DEltX2; /* single-symbol decoding */ - -size_t LIZHUF_readDTableX2 (LIZHUF_DTable* DTable, const void* src, size_t srcSize) -{ - BYTE huffWeight[LIZHUF_SYMBOLVALUE_MAX + 1]; - U32 rankVal[LIZHUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */ - U32 tableLog = 0; - U32 nbSymbols = 0; - size_t iSize; - void* const dtPtr = DTable + 1; - LIZHUF_DEltX2* const dt = (LIZHUF_DEltX2*)dtPtr; - - LIZHUF_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(LIZHUF_DTable)); - /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ - - iSize = LIZHUF_readStats(huffWeight, LIZHUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize); - if (LIZHUF_isError(iSize)) return iSize; - - /* Table header */ - { DTableDesc dtd = LIZHUF_getDTableDesc(DTable); - if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, huffman tree cannot fit in */ - dtd.tableType = 0; - dtd.tableLog = (BYTE)tableLog; - memcpy(DTable, &dtd, sizeof(dtd)); - } - - /* Prepare ranks */ - { U32 n, nextRankStart = 0; - for (n=1; n> 1; - U32 i; - LIZHUF_DEltX2 D; - D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w); - for (i = rankVal[w]; i < rankVal[w] + length; i++) - dt[i] = D; - rankVal[w] += length; - } } - - return iSize; -} - - -static BYTE LIZHUF_decodeSymbolX2(BIT_DStream_t* Dstream, const LIZHUF_DEltX2* dt, const U32 dtLog) -{ - size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ - BYTE const c = dt[val].byte; - BIT_skipBits(Dstream, dt[val].nbBits); - return c; -} - -#define LIZHUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ - *ptr++ = LIZHUF_decodeSymbolX2(DStreamPtr, dt, dtLog) - -#define LIZHUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ - if (MEM_64bits() || (LIZHUF_TABLELOG_MAX<=12)) \ - LIZHUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) - -#define LIZHUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ - if (MEM_64bits()) \ - LIZHUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) - -static inline size_t LIZHUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const LIZHUF_DEltX2* const dt, const U32 dtLog) -{ - BYTE* const pStart = p; - - /* up to 4 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-4)) { - LIZHUF_DECODE_SYMBOLX2_2(p, bitDPtr); - LIZHUF_DECODE_SYMBOLX2_1(p, bitDPtr); - LIZHUF_DECODE_SYMBOLX2_2(p, bitDPtr); - LIZHUF_DECODE_SYMBOLX2_0(p, bitDPtr); - } - - /* closer to the end */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd)) - LIZHUF_DECODE_SYMBOLX2_0(p, bitDPtr); - - /* no more data to retrieve from bitstream, hence no need to reload */ - while (p < pEnd) - LIZHUF_DECODE_SYMBOLX2_0(p, bitDPtr); - - return pEnd-pStart; -} - -static size_t LIZHUF_decompress1X2_usingDTable_internal( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const LIZHUF_DTable* DTable) -{ - BYTE* op = (BYTE*)dst; - BYTE* const oend = op + dstSize; - const void* dtPtr = DTable + 1; - const LIZHUF_DEltX2* const dt = (const LIZHUF_DEltX2*)dtPtr; - BIT_DStream_t bitD; - DTableDesc const dtd = LIZHUF_getDTableDesc(DTable); - U32 const dtLog = dtd.tableLog; - - { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); - if (LIZHUF_isError(errorCode)) return errorCode; } - - LIZHUF_decodeStreamX2(op, &bitD, oend, dt, dtLog); - - /* check */ - if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); - - return dstSize; -} - -size_t LIZHUF_decompress1X2_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const LIZHUF_DTable* DTable) -{ - DTableDesc dtd = LIZHUF_getDTableDesc(DTable); - if (dtd.tableType != 0) return ERROR(GENERIC); - return LIZHUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -} - -size_t LIZHUF_decompress1X2_DCtx (LIZHUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - const BYTE* ip = (const BYTE*) cSrc; - - size_t const hSize = LIZHUF_readDTableX2 (DCtx, cSrc, cSrcSize); - if (LIZHUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; - - return LIZHUF_decompress1X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx); -} - -size_t LIZHUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - LIZHUF_CREATE_STATIC_DTABLEX2(DTable, LIZHUF_TABLELOG_MAX); - return LIZHUF_decompress1X2_DCtx (DTable, dst, dstSize, cSrc, cSrcSize); -} - - -static size_t LIZHUF_decompress4X2_usingDTable_internal( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const LIZHUF_DTable* DTable) -{ - /* Check */ - if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ - - { const BYTE* const istart = (const BYTE*) cSrc; - BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - const void* const dtPtr = DTable + 1; - const LIZHUF_DEltX2* const dt = (const LIZHUF_DEltX2*)dtPtr; - - /* Init */ - BIT_DStream_t bitD1; - BIT_DStream_t bitD2; - BIT_DStream_t bitD3; - BIT_DStream_t bitD4; - size_t const length1 = MEM_readLE16(istart); - size_t const length2 = MEM_readLE16(istart+2); - size_t const length3 = MEM_readLE16(istart+4); - size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); - const BYTE* const istart1 = istart + 6; /* jumpTable */ - const BYTE* const istart2 = istart1 + length1; - const BYTE* const istart3 = istart2 + length2; - const BYTE* const istart4 = istart3 + length3; - const size_t segmentSize = (dstSize+3) / 4; - BYTE* const opStart2 = ostart + segmentSize; - BYTE* const opStart3 = opStart2 + segmentSize; - BYTE* const opStart4 = opStart3 + segmentSize; - BYTE* op1 = ostart; - BYTE* op2 = opStart2; - BYTE* op3 = opStart3; - BYTE* op4 = opStart4; - U32 endSignal; - DTableDesc const dtd = LIZHUF_getDTableDesc(DTable); - U32 const dtLog = dtd.tableLog; - - if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ - { size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); - if (LIZHUF_isError(errorCode)) return errorCode; } - { size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); - if (LIZHUF_isError(errorCode)) return errorCode; } - { size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); - if (LIZHUF_isError(errorCode)) return errorCode; } - { size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); - if (LIZHUF_isError(errorCode)) return errorCode; } - - /* 16-32 symbols per loop (4-8 symbols per stream) */ - endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; ) { - LIZHUF_DECODE_SYMBOLX2_2(op1, &bitD1); - LIZHUF_DECODE_SYMBOLX2_2(op2, &bitD2); - LIZHUF_DECODE_SYMBOLX2_2(op3, &bitD3); - LIZHUF_DECODE_SYMBOLX2_2(op4, &bitD4); - LIZHUF_DECODE_SYMBOLX2_1(op1, &bitD1); - LIZHUF_DECODE_SYMBOLX2_1(op2, &bitD2); - LIZHUF_DECODE_SYMBOLX2_1(op3, &bitD3); - LIZHUF_DECODE_SYMBOLX2_1(op4, &bitD4); - LIZHUF_DECODE_SYMBOLX2_2(op1, &bitD1); - LIZHUF_DECODE_SYMBOLX2_2(op2, &bitD2); - LIZHUF_DECODE_SYMBOLX2_2(op3, &bitD3); - LIZHUF_DECODE_SYMBOLX2_2(op4, &bitD4); - LIZHUF_DECODE_SYMBOLX2_0(op1, &bitD1); - LIZHUF_DECODE_SYMBOLX2_0(op2, &bitD2); - LIZHUF_DECODE_SYMBOLX2_0(op3, &bitD3); - LIZHUF_DECODE_SYMBOLX2_0(op4, &bitD4); - endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - } - - /* check corruption */ - if (op1 > opStart2) return ERROR(corruption_detected); - if (op2 > opStart3) return ERROR(corruption_detected); - if (op3 > opStart4) return ERROR(corruption_detected); - /* note : op4 supposed already verified within main loop */ - - /* finish bitStreams one by one */ - LIZHUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); - LIZHUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); - LIZHUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); - LIZHUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); - - /* check */ - endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); - if (!endSignal) return ERROR(corruption_detected); - - /* decoded size */ - return dstSize; - } -} - - -size_t LIZHUF_decompress4X2_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const LIZHUF_DTable* DTable) -{ - DTableDesc dtd = LIZHUF_getDTableDesc(DTable); - if (dtd.tableType != 0) return ERROR(GENERIC); - return LIZHUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -} - - -size_t LIZHUF_decompress4X2_DCtx (LIZHUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - const BYTE* ip = (const BYTE*) cSrc; - - size_t const hSize = LIZHUF_readDTableX2 (dctx, cSrc, cSrcSize); - if (LIZHUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; - - return LIZHUF_decompress4X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, dctx); -} - -size_t LIZHUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - LIZHUF_CREATE_STATIC_DTABLEX2(DTable, LIZHUF_TABLELOG_MAX); - return LIZHUF_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} - - -/* *************************/ -/* double-symbols decoding */ -/* *************************/ -typedef struct { U16 sequence; BYTE nbBits; BYTE length; } LIZHUF_DEltX4; /* double-symbols decoding */ - -typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; - -static void LIZHUF_fillDTableX4Level2(LIZHUF_DEltX4* DTable, U32 sizeLog, const U32 consumed, - const U32* rankValOrigin, const int minWeight, - const sortedSymbol_t* sortedSymbols, const U32 sortedListSize, - U32 nbBitsBaseline, U16 baseSeq) -{ - LIZHUF_DEltX4 DElt; - U32 rankVal[LIZHUF_TABLELOG_ABSOLUTEMAX + 1]; - - /* get pre-calculated rankVal */ - memcpy(rankVal, rankValOrigin, sizeof(rankVal)); - - /* fill skipped values */ - if (minWeight>1) { - U32 i, skipSize = rankVal[minWeight]; - MEM_writeLE16(&(DElt.sequence), baseSeq); - DElt.nbBits = (BYTE)(consumed); - DElt.length = 1; - for (i = 0; i < skipSize; i++) - DTable[i] = DElt; - } - - /* fill DTable */ - { U32 s; for (s=0; s= 1 */ - - rankVal[weight] += length; - } } -} - -typedef U32 rankVal_t[LIZHUF_TABLELOG_ABSOLUTEMAX][LIZHUF_TABLELOG_ABSOLUTEMAX + 1]; - -static void LIZHUF_fillDTableX4(LIZHUF_DEltX4* DTable, const U32 targetLog, - const sortedSymbol_t* sortedList, const U32 sortedListSize, - const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight, - const U32 nbBitsBaseline) -{ - U32 rankVal[LIZHUF_TABLELOG_ABSOLUTEMAX + 1]; - const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ - const U32 minBits = nbBitsBaseline - maxWeight; - U32 s; - - memcpy(rankVal, rankValOrigin, sizeof(rankVal)); - - /* fill DTable */ - for (s=0; s= minBits) { /* enough room for a second symbol */ - U32 sortedRank; - int minWeight = nbBits + scaleLog; - if (minWeight < 1) minWeight = 1; - sortedRank = rankStart[minWeight]; - LIZHUF_fillDTableX4Level2(DTable+start, targetLog-nbBits, nbBits, - rankValOrigin[nbBits], minWeight, - sortedList+sortedRank, sortedListSize-sortedRank, - nbBitsBaseline, symbol); - } else { - LIZHUF_DEltX4 DElt; - MEM_writeLE16(&(DElt.sequence), symbol); - DElt.nbBits = (BYTE)(nbBits); - DElt.length = 1; - { U32 const end = start + length; - U32 u; - for (u = start; u < end; u++) DTable[u] = DElt; - } } - rankVal[weight] += length; - } -} - -size_t LIZHUF_readDTableX4 (LIZHUF_DTable* DTable, const void* src, size_t srcSize) -{ - BYTE weightList[LIZHUF_SYMBOLVALUE_MAX + 1]; - sortedSymbol_t sortedSymbol[LIZHUF_SYMBOLVALUE_MAX + 1]; - U32 rankStats[LIZHUF_TABLELOG_ABSOLUTEMAX + 1] = { 0 }; - U32 rankStart0[LIZHUF_TABLELOG_ABSOLUTEMAX + 2] = { 0 }; - U32* const rankStart = rankStart0+1; - rankVal_t rankVal; - U32 tableLog, maxW, sizeOfSort, nbSymbols; - DTableDesc dtd = LIZHUF_getDTableDesc(DTable); - U32 const maxTableLog = dtd.maxTableLog; - size_t iSize; - void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ - LIZHUF_DEltX4* const dt = (LIZHUF_DEltX4*)dtPtr; - - LIZHUF_STATIC_ASSERT(sizeof(LIZHUF_DEltX4) == sizeof(LIZHUF_DTable)); /* if compilation fails here, assertion is false */ - if (maxTableLog > LIZHUF_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge); - /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ - - iSize = LIZHUF_readStats(weightList, LIZHUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); - if (LIZHUF_isError(iSize)) return iSize; - - /* check result */ - if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ - - /* find maxWeight */ - for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ - - /* Get start index of each weight */ - { U32 w, nextRankStart = 0; - for (w=1; w> consumed; - } } } } - - LIZHUF_fillDTableX4(dt, maxTableLog, - sortedSymbol, sizeOfSort, - rankStart0, rankVal, maxW, - tableLog+1); - - dtd.tableLog = (BYTE)maxTableLog; - dtd.tableType = 1; - memcpy(DTable, &dtd, sizeof(dtd)); - return iSize; -} - - -static U32 LIZHUF_decodeSymbolX4(void* op, BIT_DStream_t* DStream, const LIZHUF_DEltX4* dt, const U32 dtLog) -{ - size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ - memcpy(op, dt+val, 2); - BIT_skipBits(DStream, dt[val].nbBits); - return dt[val].length; -} - -static U32 LIZHUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const LIZHUF_DEltX4* dt, const U32 dtLog) -{ - size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ - memcpy(op, dt+val, 1); - if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits); - else { - if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { - BIT_skipBits(DStream, dt[val].nbBits); - if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) - DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ - } } - return 1; -} - - -#define LIZHUF_DECODE_SYMBOLX4_0(ptr, DStreamPtr) \ - ptr += LIZHUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) - -#define LIZHUF_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \ - if (MEM_64bits() || (LIZHUF_TABLELOG_MAX<=12)) \ - ptr += LIZHUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) - -#define LIZHUF_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \ - if (MEM_64bits()) \ - ptr += LIZHUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) - -static inline size_t LIZHUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const LIZHUF_DEltX4* const dt, const U32 dtLog) -{ - BYTE* const pStart = p; - - /* up to 8 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) { - LIZHUF_DECODE_SYMBOLX4_2(p, bitDPtr); - LIZHUF_DECODE_SYMBOLX4_1(p, bitDPtr); - LIZHUF_DECODE_SYMBOLX4_2(p, bitDPtr); - LIZHUF_DECODE_SYMBOLX4_0(p, bitDPtr); - } - - /* closer to end : up to 2 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2)) - LIZHUF_DECODE_SYMBOLX4_0(p, bitDPtr); - - while (p <= pEnd-2) - LIZHUF_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ - - if (p < pEnd) - p += LIZHUF_decodeLastSymbolX4(p, bitDPtr, dt, dtLog); - - return p-pStart; -} - - -static size_t LIZHUF_decompress1X4_usingDTable_internal( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const LIZHUF_DTable* DTable) -{ - BIT_DStream_t bitD; - - /* Init */ - { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); - if (LIZHUF_isError(errorCode)) return errorCode; - } - - /* decode */ - { BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ - const LIZHUF_DEltX4* const dt = (const LIZHUF_DEltX4*)dtPtr; - DTableDesc const dtd = LIZHUF_getDTableDesc(DTable); - LIZHUF_decodeStreamX4(ostart, &bitD, oend, dt, dtd.tableLog); - } - - /* check */ - if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); - - /* decoded size */ - return dstSize; -} - -size_t LIZHUF_decompress1X4_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const LIZHUF_DTable* DTable) -{ - DTableDesc dtd = LIZHUF_getDTableDesc(DTable); - if (dtd.tableType != 1) return ERROR(GENERIC); - return LIZHUF_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -} - -size_t LIZHUF_decompress1X4_DCtx (LIZHUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - const BYTE* ip = (const BYTE*) cSrc; - - size_t const hSize = LIZHUF_readDTableX4 (DCtx, cSrc, cSrcSize); - if (LIZHUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; - - return LIZHUF_decompress1X4_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx); -} - -size_t LIZHUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - LIZHUF_CREATE_STATIC_DTABLEX4(DTable, LIZHUF_TABLELOG_MAX); - return LIZHUF_decompress1X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} - -static size_t LIZHUF_decompress4X4_usingDTable_internal( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const LIZHUF_DTable* DTable) -{ - if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ - - { const BYTE* const istart = (const BYTE*) cSrc; - BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - const void* const dtPtr = DTable+1; - const LIZHUF_DEltX4* const dt = (const LIZHUF_DEltX4*)dtPtr; - - /* Init */ - BIT_DStream_t bitD1; - BIT_DStream_t bitD2; - BIT_DStream_t bitD3; - BIT_DStream_t bitD4; - size_t const length1 = MEM_readLE16(istart); - size_t const length2 = MEM_readLE16(istart+2); - size_t const length3 = MEM_readLE16(istart+4); - size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); - const BYTE* const istart1 = istart + 6; /* jumpTable */ - const BYTE* const istart2 = istart1 + length1; - const BYTE* const istart3 = istart2 + length2; - const BYTE* const istart4 = istart3 + length3; - size_t const segmentSize = (dstSize+3) / 4; - BYTE* const opStart2 = ostart + segmentSize; - BYTE* const opStart3 = opStart2 + segmentSize; - BYTE* const opStart4 = opStart3 + segmentSize; - BYTE* op1 = ostart; - BYTE* op2 = opStart2; - BYTE* op3 = opStart3; - BYTE* op4 = opStart4; - U32 endSignal; - DTableDesc const dtd = LIZHUF_getDTableDesc(DTable); - U32 const dtLog = dtd.tableLog; - - if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ - { size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); - if (LIZHUF_isError(errorCode)) return errorCode; } - { size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); - if (LIZHUF_isError(errorCode)) return errorCode; } - { size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); - if (LIZHUF_isError(errorCode)) return errorCode; } - { size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); - if (LIZHUF_isError(errorCode)) return errorCode; } - - /* 16-32 symbols per loop (4-8 symbols per stream) */ - endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - for ( ; (endSignal==BIT_DStream_unfinished) & (op4<(oend-(sizeof(bitD4.bitContainer)-1))) ; ) { - LIZHUF_DECODE_SYMBOLX4_2(op1, &bitD1); - LIZHUF_DECODE_SYMBOLX4_2(op2, &bitD2); - LIZHUF_DECODE_SYMBOLX4_2(op3, &bitD3); - LIZHUF_DECODE_SYMBOLX4_2(op4, &bitD4); - LIZHUF_DECODE_SYMBOLX4_1(op1, &bitD1); - LIZHUF_DECODE_SYMBOLX4_1(op2, &bitD2); - LIZHUF_DECODE_SYMBOLX4_1(op3, &bitD3); - LIZHUF_DECODE_SYMBOLX4_1(op4, &bitD4); - LIZHUF_DECODE_SYMBOLX4_2(op1, &bitD1); - LIZHUF_DECODE_SYMBOLX4_2(op2, &bitD2); - LIZHUF_DECODE_SYMBOLX4_2(op3, &bitD3); - LIZHUF_DECODE_SYMBOLX4_2(op4, &bitD4); - LIZHUF_DECODE_SYMBOLX4_0(op1, &bitD1); - LIZHUF_DECODE_SYMBOLX4_0(op2, &bitD2); - LIZHUF_DECODE_SYMBOLX4_0(op3, &bitD3); - LIZHUF_DECODE_SYMBOLX4_0(op4, &bitD4); - - endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - } - - /* check corruption */ - if (op1 > opStart2) return ERROR(corruption_detected); - if (op2 > opStart3) return ERROR(corruption_detected); - if (op3 > opStart4) return ERROR(corruption_detected); - /* note : op4 already verified within main loop */ - - /* finish bitStreams one by one */ - LIZHUF_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog); - LIZHUF_decodeStreamX4(op2, &bitD2, opStart3, dt, dtLog); - LIZHUF_decodeStreamX4(op3, &bitD3, opStart4, dt, dtLog); - LIZHUF_decodeStreamX4(op4, &bitD4, oend, dt, dtLog); - - /* check */ - { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); - if (!endCheck) return ERROR(corruption_detected); } - - /* decoded size */ - return dstSize; - } -} - - -size_t LIZHUF_decompress4X4_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const LIZHUF_DTable* DTable) -{ - DTableDesc dtd = LIZHUF_getDTableDesc(DTable); - if (dtd.tableType != 1) return ERROR(GENERIC); - return LIZHUF_decompress4X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -} - - -size_t LIZHUF_decompress4X4_DCtx (LIZHUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - const BYTE* ip = (const BYTE*) cSrc; - - size_t hSize = LIZHUF_readDTableX4 (dctx, cSrc, cSrcSize); - if (LIZHUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; - - return LIZHUF_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx); -} - -size_t LIZHUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - LIZHUF_CREATE_STATIC_DTABLEX4(DTable, LIZHUF_TABLELOG_MAX); - return LIZHUF_decompress4X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} - - -/* ********************************/ -/* Generic decompression selector */ -/* ********************************/ - -size_t LIZHUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, - const void* cSrc, size_t cSrcSize, - const LIZHUF_DTable* DTable) -{ - DTableDesc const dtd = LIZHUF_getDTableDesc(DTable); - return dtd.tableType ? LIZHUF_decompress1X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) : - LIZHUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); -} - -size_t LIZHUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, - const void* cSrc, size_t cSrcSize, - const LIZHUF_DTable* DTable) -{ - DTableDesc const dtd = LIZHUF_getDTableDesc(DTable); - return dtd.tableType ? LIZHUF_decompress4X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) : - LIZHUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); -} - - -typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; -static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = -{ - /* single, double, quad */ - {{0,0}, {1,1}, {2,2}}, /* Q==0 : impossible */ - {{0,0}, {1,1}, {2,2}}, /* Q==1 : impossible */ - {{ 38,130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */ - {{ 448,128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */ - {{ 556,128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */ - {{ 714,128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */ - {{ 883,128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */ - {{ 897,128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */ - {{ 926,128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */ - {{ 947,128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */ - {{1107,128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */ - {{1177,128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */ - {{1242,128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */ - {{1349,128}, {2644,106}, {5260,106}}, /* Q ==13 : 81-87% */ - {{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */ - {{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */ -}; - -/** LIZHUF_selectDecoder() : -* Tells which decoder is likely to decode faster, -* based on a set of pre-determined metrics. -* @return : 0==LIZHUF_decompress4X2, 1==LIZHUF_decompress4X4 . -* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ -U32 LIZHUF_selectDecoder (size_t dstSize, size_t cSrcSize) -{ - /* decoder timing evaluation */ - U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */ - U32 const D256 = (U32)(dstSize >> 8); - U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); - U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); - DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, for cache eviction */ - - return DTime1 < DTime0; -} - - -typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); - -size_t LIZHUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - static const decompressionAlgo decompress[2] = { LIZHUF_decompress4X2, LIZHUF_decompress4X4 }; - - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ - - { U32 const algoNb = LIZHUF_selectDecoder(dstSize, cSrcSize); - return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); - } -} - -size_t LIZHUF_decompress4X_DCtx (LIZHUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ - - { U32 const algoNb = LIZHUF_selectDecoder(dstSize, cSrcSize); - return algoNb ? LIZHUF_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : - LIZHUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; - } -} - -size_t LIZHUF_decompress4X_hufOnly (LIZHUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if ((cSrcSize >= dstSize) || (cSrcSize <= 1)) return ERROR(corruption_detected); /* invalid */ - - { U32 const algoNb = LIZHUF_selectDecoder(dstSize, cSrcSize); - return algoNb ? LIZHUF_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : - LIZHUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; - } -} - -size_t LIZHUF_decompress1X_DCtx (LIZHUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ - - { U32 const algoNb = LIZHUF_selectDecoder(dstSize, cSrcSize); - return algoNb ? LIZHUF_decompress1X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : - LIZHUF_decompress1X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; - } -} diff --git a/C/lizard/liz_lz5_compress.c b/C/lizard/liz_lz5_compress.c deleted file mode 100644 index a93546f5..00000000 --- a/C/lizard/liz_lz5_compress.c +++ /dev/null @@ -1,637 +0,0 @@ -/* - LZ5 - Fast LZ compression algorithm - Copyright (C) 2011-2015, Yann Collet. - Copyright (C) 2015-2016, Przemyslaw Skibinski - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOTLIZ_hash4Ptr - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ5 source repository : https://github.com/inikep/lz5 -*/ - - -/* ************************************* -* Includes -***************************************/ -#include "liz_compress.h" -#include "liz_common.h" -#include -#include // intptr_t -#ifndef USE_LZ4_ONLY - #ifdef LIZ_USE_TEST - #include "test/lz5_common_test.h" - #include "test/lz5_compress_test.h" - #else - #include "liz_compress_lz5v2.h" - #endif -#endif -#include "liz_compress_lz4.h" -#include "huf.h" - - -/* ************************************* -* Local Macros -***************************************/ -#define DELTANEXT(p) chainTable[(p) & contentMask] -#define LIZ_MINIMAL_HUFF_GAIN(comprSize) (comprSize + (comprSize/8) + 512) -#define LIZ_MINIMAL_BLOCK_GAIN(comprSize) (comprSize + (comprSize/32) + 512) - - -/*-************************************ -* Local Utils -**************************************/ -int LIZ_versionNumber (void) { return LIZ_VERSION_NUMBER; } -int LIZ_compressBound(int isize) { return LIZ_COMPRESSBOUND(isize); } -int LIZ_sizeofState_MinLevel() { return LIZ_sizeofState(LIZ_MIN_CLEVEL); } - - - -/* ************************************* -* Hash functions -***************************************/ -#define HASH_UPDATE_LIMIT 8 /* equal to MEM_read64 */ -static const U32 prime4bytes = 2654435761U; -static const U64 prime5bytes = 889523592379ULL; -static const U64 prime6bytes = 227718039650203ULL; -static const U64 prime7bytes = 58295818150454627ULL; - -#if MINMATCH == 3 -static const U32 prime3bytes = 506832829U; -static U32 LIZ_hash3(U32 u, U32 h) { return (u * prime3bytes) << (32-24) >> (32-h) ; } -static size_t LIZ_hash3Ptr(const void* ptr, U32 h) { return LIZ_hash3(MEM_read32(ptr), h); } -#endif - -static U32 LIZ_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; } -static size_t LIZ_hash4Ptr(const void* ptr, U32 h) { return LIZ_hash4(MEM_read32(ptr), h); } - -static size_t LIZ_hash5(U64 u, U32 h) { return (size_t)((u * prime5bytes) << (64-40) >> (64-h)) ; } -static size_t LIZ_hash5Ptr(const void* p, U32 h) { return LIZ_hash5(MEM_read64(p), h); } - -static size_t LIZ_hash6(U64 u, U32 h) { return (size_t)((u * prime6bytes) << (64-48) >> (64-h)) ; } -static size_t LIZ_hash6Ptr(const void* p, U32 h) { return LIZ_hash6(MEM_read64(p), h); } - -static size_t LIZ_hash7(U64 u, U32 h) { return (size_t)((u * prime7bytes) << (64-56) >> (64-h)) ; } -static size_t LIZ_hash7Ptr(const void* p, U32 h) { return LIZ_hash7(MEM_read64(p), h); } - -static size_t LIZ_hashPtr(const void* p, U32 hBits, U32 mls) -{ - switch(mls) - { - default: - case 4: return LIZ_hash4Ptr(p, hBits); - case 5: return LIZ_hash5Ptr(p, hBits); - case 6: return LIZ_hash6Ptr(p, hBits); - case 7: return LIZ_hash7Ptr(p, hBits); - } -} - - - - -/************************************** -* Internal functions -**************************************/ -/** LIZ_count_2segments() : -* can count match length with `ip` & `match` in 2 different segments. -* convention : on reaching mEnd, match count continue starting from iStart -*/ -static size_t LIZ_count_2segments(const BYTE* ip, const BYTE* match, const BYTE* iEnd, const BYTE* mEnd, const BYTE* iStart) -{ - const BYTE* const vEnd = MIN( ip + (mEnd - match), iEnd); - size_t const matchLength = LIZ_count(ip, match, vEnd); - if (match + matchLength != mEnd) return matchLength; - return matchLength + LIZ_count(ip+matchLength, iStart, iEnd); -} - - -void LIZ_initBlock(LIZ_stream_t* ctx) -{ - ctx->offset16Ptr = ctx->offset16Base; - ctx->offset24Ptr = ctx->offset24Base; - ctx->lenPtr = ctx->lenBase; - ctx->literalsPtr = ctx->literalsBase; - ctx->flagsPtr = ctx->flagsBase; - ctx->last_off = LIZ_INIT_LAST_OFFSET; /* reset last offset */ -} - - -FORCE_INLINE int LIZ_writeStream(int useHuff, LIZ_stream_t* ctx, BYTE* streamPtr, uint32_t streamLen, BYTE** op, BYTE* oend) -{ - if (useHuff && streamLen > 1024) { -#ifndef LIZ_NO_HUFFMAN - int useHuffBuf; - if (*op + 6 > oend) { LIZ_LOG_COMPRESS("*op[%p] + 6 > oend[%p]\n", *op, oend); return -1; } - - useHuffBuf = ((size_t)(oend - (*op + 6)) < LIZHUF_compressBound(streamLen)); - if (useHuffBuf) { - if (streamLen > LIZ_BLOCK_SIZE) { LIZ_LOG_COMPRESS("streamLen[%d] > LIZ_BLOCK_SIZE\n", streamLen); return -1; } - ctx->comprStreamLen = (U32)LIZHUF_compress(ctx->huffBase, ctx->huffEnd - ctx->huffBase, streamPtr, streamLen); - } else { - ctx->comprStreamLen = (U32)LIZHUF_compress(*op + 6, oend - (*op + 6), streamPtr, streamLen); - } - - if (!LIZHUF_isError(ctx->comprStreamLen)) { - if (ctx->comprStreamLen > 0 && (LIZ_MINIMAL_HUFF_GAIN(ctx->comprStreamLen) < streamLen)) { /* compressible */ - MEM_writeLE24(*op, streamLen); - MEM_writeLE24(*op+3, ctx->comprStreamLen); - if (useHuffBuf) { - if ((size_t)(oend - (*op + 6)) < ctx->comprStreamLen) { LIZ_LOG_COMPRESS("*op[%p] oend[%p] comprStreamLen[%d]\n", *op, oend, (int)ctx->comprStreamLen); return -1; } - memcpy(*op + 6, ctx->huffBase, ctx->comprStreamLen); - } - *op += ctx->comprStreamLen + 6; - LIZ_LOG_COMPRESS("LIZHUF_compress streamLen=%d comprStreamLen=%d\n", (int)streamLen, (int)ctx->comprStreamLen); - return 1; - } else { LIZ_LOG_COMPRESS("LIZHUF_compress ERROR comprStreamLen=%d streamLen=%d\n", (int)ctx->comprStreamLen, (int)streamLen); } - } else { LIZ_LOG_COMPRESS("LIZHUF_compress ERROR %d: %s\n", (int)ctx->comprStreamLen, LIZHUF_getErrorName(ctx->comprStreamLen)); } -#else - LIZ_LOG_COMPRESS("compiled with LIZ_NO_HUFFMAN\n"); - (void)ctx; - return -1; -#endif - } else ctx->comprStreamLen = 0; - - if (*op + 3 + streamLen > oend) { LIZ_LOG_COMPRESS("*op[%p] + 3 + streamLen[%d] > oend[%p]\n", *op, streamLen, oend); return -1; } - MEM_writeLE24(*op, streamLen); - *op += 3; - memcpy(*op, streamPtr, streamLen); - *op += streamLen; - LIZ_LOG_COMPRESS("Uncompressed streamLen=%d\n", (int)streamLen); - return 0; -} - - -int LIZ_writeBlock(LIZ_stream_t* ctx, const BYTE* ip, uint32_t inputSize, BYTE** op, BYTE* oend) -{ - int res; - uint32_t flagsLen = (uint32_t)(ctx->flagsPtr - ctx->flagsBase); - uint32_t literalsLen = (uint32_t)(ctx->literalsPtr - ctx->literalsBase); - uint32_t lenLen = (uint32_t)(ctx->lenPtr - ctx->lenBase); - uint32_t offset16Len = (uint32_t)(ctx->offset16Ptr - ctx->offset16Base); - uint32_t offset24Len = (uint32_t)(ctx->offset24Ptr - ctx->offset24Base); - uint32_t sum = flagsLen + literalsLen + lenLen + offset16Len + offset24Len; -#ifdef LIZ_USE_LOGS - uint32_t comprFlagsLen, comprLiteralsLen; -#endif - - BYTE* start = *op; - - if ((literalsLen < WILDCOPYLENGTH) || (sum+5*3+1 > inputSize)) goto _write_uncompressed; - - *start = 0; - *op += 1; - - res = LIZ_writeStream(0, ctx, ctx->lenBase, lenLen, op, oend); - if (res < 0) goto _output_error; else *start += (BYTE)(res*LIZ_FLAG_LEN); - - res = LIZ_writeStream(ctx->huffType&LIZ_FLAG_OFFSET16, ctx, ctx->offset16Base, offset16Len, op, oend); - if (res < 0) goto _output_error; else *start += (BYTE)(res*LIZ_FLAG_OFFSET16); - - res = LIZ_writeStream(ctx->huffType&LIZ_FLAG_OFFSET24, ctx, ctx->offset24Base, offset24Len, op, oend); - if (res < 0) goto _output_error; else *start += (BYTE)(res*LIZ_FLAG_OFFSET24); - - res = LIZ_writeStream(ctx->huffType&LIZ_FLAG_FLAGS, ctx, ctx->flagsBase, flagsLen, op, oend); - if (res < 0) goto _output_error; else *start += (BYTE)(res*LIZ_FLAG_FLAGS); -#ifdef LIZ_USE_LOGS - comprFlagsLen = ctx->comprStreamLen; -#endif - - res = LIZ_writeStream(ctx->huffType&LIZ_FLAG_LITERALS, ctx, ctx->literalsBase, literalsLen, op, oend); - if (res < 0) goto _output_error; else *start += (BYTE)(res*LIZ_FLAG_LITERALS); -#ifdef LIZ_USE_LOGS - comprLiteralsLen = ctx->comprStreamLen; - sum = (int)(*op-start); -#endif - - if (LIZ_MINIMAL_BLOCK_GAIN((uint32_t)(*op-start)) > inputSize) goto _write_uncompressed; - - LIZ_LOG_COMPRESS("%d: total=%d block=%d flagsLen[%.2f%%]=%d comprFlagsLen[%.2f%%]=%d literalsLen[%.2f%%]=%d comprLiteralsLen[%.2f%%]=%d lenLen=%d offset16Len[%.2f%%]=%d offset24Len[%.2f%%]=%d\n", (int)(ip - ctx->srcBase), - (int)(*op - ctx->destBase), sum, (flagsLen*100.0)/sum, flagsLen, (comprFlagsLen*100.0)/sum, comprFlagsLen, (literalsLen*100.0)/sum, literalsLen, (comprLiteralsLen*100.0)/sum, comprLiteralsLen, - lenLen, (offset16Len*100.0)/sum, offset16Len, (offset24Len*100.0)/sum, offset24Len); - return 0; - -_write_uncompressed: - LIZ_LOG_COMPRESS("%d: total=%d block=%d UNCOMPRESSED inputSize=%u outSize=%d\n", (int)(ip - ctx->srcBase), - (int)(*op - ctx->destBase), (int)(*op-start), inputSize, (int)(oend-start)); - if ((uint32_t)(oend - start) < inputSize + 4) goto _output_error; - *start = LIZ_FLAG_UNCOMPRESSED; - *op = start + 1; - MEM_writeLE24(*op, inputSize); - *op += 3; - memcpy(*op, ip, inputSize); - *op += inputSize; - return 0; - -_output_error: - LIZ_LOG_COMPRESS("LIZ_writeBlock ERROR size=%d/%d flagsLen=%d literalsLen=%d lenLen=%d offset16Len=%d offset24Len=%d\n", (int)(*op-start), (int)(oend-start), flagsLen, literalsLen, lenLen, offset16Len, offset24Len); - return 1; -} - - -FORCE_INLINE int LIZ_encodeSequence ( - LIZ_stream_t* ctx, - const BYTE** ip, - const BYTE** anchor, - size_t matchLength, - const BYTE* const match) -{ -#ifdef USE_LZ4_ONLY - return LIZ_encodeSequence_LZ4(ctx, ip, anchor, matchLength, match); -#else - if (ctx->params.decompressType == LIZ_coderwords_LZ4) - return LIZ_encodeSequence_LZ4(ctx, ip, anchor, matchLength, match); - - return LIZ_encodeSequence_LZ5v2(ctx, ip, anchor, matchLength, match); -#endif -} - - -FORCE_INLINE int LIZ_encodeLastLiterals ( - LIZ_stream_t* ctx, - const BYTE** ip, - const BYTE** anchor) -{ - LIZ_LOG_COMPRESS("LIZ_encodeLastLiterals LIZ_coderwords_LZ4=%d\n", ctx->params.decompressType == LIZ_coderwords_LZ4); -#ifdef USE_LZ4_ONLY - return LIZ_encodeLastLiterals_LZ4(ctx, ip, anchor); -#else - if (ctx->params.decompressType == LIZ_coderwords_LZ4) - return LIZ_encodeLastLiterals_LZ4(ctx, ip, anchor); - - return LIZ_encodeLastLiterals_LZ5v2(ctx, ip, anchor); -#endif -} - - -/************************************** -* Include parsers -**************************************/ -#include "liz_parser_hashchain.h" -#include "liz_parser_nochain.h" -#include "liz_parser_fast.h" -#include "liz_parser_fastsmall.h" -#include "liz_parser_fastbig.h" -#ifndef USE_LZ4_ONLY - #include "liz_parser_optimal.h" - #include "liz_parser_lowestprice.h" - #include "liz_parser_pricefast.h" -#endif - - -int LIZ_verifyCompressionLevel(int compressionLevel) -{ - (void)LIZ_hashPtr; - (void)LIZ_wildCopy16; - if (compressionLevel > LIZ_MAX_CLEVEL) compressionLevel = LIZ_MAX_CLEVEL; - if (compressionLevel < LIZ_MIN_CLEVEL) compressionLevel = LIZ_DEFAULT_CLEVEL; - return compressionLevel; -} - - -int LIZ_sizeofState(int compressionLevel) -{ - LIZ_parameters params; - U32 hashTableSize, chainTableSize; - - compressionLevel = LIZ_verifyCompressionLevel(compressionLevel); - params = LIZ_defaultParameters[compressionLevel - LIZ_MIN_CLEVEL]; -// hashTableSize = (U32)(sizeof(U32)*(((size_t)1 << params.hashLog3)+((size_t)1 << params.hashLog))); - hashTableSize = (U32)(sizeof(U32)*(((size_t)1 << params.hashLog))); - chainTableSize = (U32)(sizeof(U32)*((size_t)1 << params.contentLog)); - - return sizeof(LIZ_stream_t) + hashTableSize + chainTableSize + LIZ_COMPRESS_ADD_BUF + (int)LIZ_COMPRESS_ADD_HUF; -} - - -static void LIZ_init(LIZ_stream_t* ctx, const BYTE* start) -{ - MEM_INIT((void*)ctx->hashTable, 0, ctx->hashTableSize); - MEM_INIT(ctx->chainTable, 0x01, ctx->chainTableSize); - // printf("memset hashTable=%p hashEnd=%p chainTable=%p chainEnd=%p\n", ctx->hashTable, ((BYTE*)ctx->hashTable) + ctx->hashTableSize, ctx->chainTable, ((BYTE*)ctx->chainTable)+ctx->chainTableSize); - ctx->nextToUpdate = LIZ_DICT_SIZE; - ctx->base = start - LIZ_DICT_SIZE; - ctx->end = start; - ctx->dictBase = start - LIZ_DICT_SIZE; - ctx->dictLimit = LIZ_DICT_SIZE; - ctx->lowLimit = LIZ_DICT_SIZE; - ctx->last_off = LIZ_INIT_LAST_OFFSET; - ctx->litSum = 0; -} - - -/* if ctx==NULL memory is allocated and returned as value */ -LIZ_stream_t* LIZ_initStream(LIZ_stream_t* ctx, int compressionLevel) -{ - LIZ_parameters params; - U32 hashTableSize, chainTableSize; - void *tempPtr; - - compressionLevel = LIZ_verifyCompressionLevel(compressionLevel); - params = LIZ_defaultParameters[compressionLevel - LIZ_MIN_CLEVEL]; -// hashTableSize = (U32)(sizeof(U32)*(((size_t)1 << params.hashLog3)+((size_t)1 << params.hashLog))); - hashTableSize = (U32)(sizeof(U32)*(((size_t)1 << params.hashLog))); - chainTableSize = (U32)(sizeof(U32)*((size_t)1 << params.contentLog)); - - if (!ctx) - { - ctx = (LIZ_stream_t*)malloc(sizeof(LIZ_stream_t) + hashTableSize + chainTableSize + LIZ_COMPRESS_ADD_BUF + LIZ_COMPRESS_ADD_HUF); - if (!ctx) { printf("ERROR: Cannot allocate %d MB (compressionLevel=%d)\n", (int)(sizeof(LIZ_stream_t) + hashTableSize + chainTableSize)>>20, compressionLevel); return 0; } - LIZ_LOG_COMPRESS("Allocated %d MB (compressionLevel=%d)\n", (int)(sizeof(LIZ_stream_t) + hashTableSize + chainTableSize)>>20, compressionLevel); - ctx->allocatedMemory = sizeof(LIZ_stream_t) + hashTableSize + chainTableSize + LIZ_COMPRESS_ADD_BUF + (U32)LIZ_COMPRESS_ADD_HUF; - // printf("malloc from=%p to=%p hashTable=%p hashEnd=%p chainTable=%p chainEnd=%p\n", ctx, ((BYTE*)ctx)+sizeof(LIZ_stream_t) + hashTableSize + chainTableSize, ctx->hashTable, ((BYTE*)ctx->hashTable) + hashTableSize, ctx->chainTable, ((BYTE*)ctx->chainTable)+chainTableSize); - } - - tempPtr = ctx; - ctx->hashTable = (U32*)(tempPtr) + sizeof(LIZ_stream_t)/4; - ctx->hashTableSize = hashTableSize; - ctx->chainTable = ctx->hashTable + hashTableSize/4; - ctx->chainTableSize = chainTableSize; - ctx->params = params; - ctx->compressionLevel = (unsigned)compressionLevel; - if (compressionLevel < 30) - ctx->huffType = 0; - else - ctx->huffType = LIZ_FLAG_LITERALS + LIZ_FLAG_FLAGS; // + LIZ_FLAG_OFFSET16 + LIZ_FLAG_OFFSET24; - - ctx->literalsBase = (BYTE*)ctx->hashTable + ctx->hashTableSize + ctx->chainTableSize; - ctx->flagsBase = ctx->literalsEnd = ctx->literalsBase + LIZ_BLOCK_SIZE_PAD; - ctx->lenBase = ctx->flagsEnd = ctx->flagsBase + LIZ_BLOCK_SIZE_PAD; - ctx->offset16Base = ctx->lenEnd = ctx->lenBase + LIZ_BLOCK_SIZE_PAD; - ctx->offset24Base = ctx->offset16End = ctx->offset16Base + LIZ_BLOCK_SIZE_PAD; - ctx->huffBase = ctx->offset24End = ctx->offset24Base + LIZ_BLOCK_SIZE_PAD; - ctx->huffEnd = ctx->huffBase + LIZ_COMPRESS_ADD_HUF; - - return ctx; -} - - - -LIZ_stream_t* LIZ_createStream(int compressionLevel) -{ - LIZ_stream_t* ctx = LIZ_initStream(NULL, compressionLevel); -// if (ctx) printf("LIZ_createStream ctx=%p ctx->compressionLevel=%d\n", ctx, ctx->compressionLevel); - return ctx; -} - - -/* initialization */ -LIZ_stream_t* LIZ_resetStream(LIZ_stream_t* ctx, int compressionLevel) -{ - size_t wanted = LIZ_sizeofState(compressionLevel); - -// printf("LIZ_resetStream ctx=%p cLevel=%d have=%d wanted=%d min=%d\n", ctx, compressionLevel, (int)have, (int)wanted, (int)sizeof(LIZ_stream_t)); - if (ctx->allocatedMemory < wanted) - { - // printf("REALLOC ctx=%p cLevel=%d have=%d wanted=%d\n", ctx, compressionLevel, (int)have, (int)wanted); - LIZ_freeStream(ctx); - ctx = LIZ_createStream(compressionLevel); - } - else - { - LIZ_initStream(ctx, compressionLevel); - } - - if (ctx) ctx->base = NULL; - return ctx; -} - - -int LIZ_freeStream(LIZ_stream_t* ctx) -{ - if (ctx) { - // printf("LIZ_freeStream ctx=%p ctx->compressionLevel=%d\n", ctx, ctx->compressionLevel); - free(ctx); - } - return 0; -} - - -int LIZ_loadDict(LIZ_stream_t* LIZ_streamPtr, const char* dictionary, int dictSize) -{ - LIZ_stream_t* ctxPtr = (LIZ_stream_t*) LIZ_streamPtr; - if (dictSize > LIZ_DICT_SIZE) { - dictionary += dictSize - LIZ_DICT_SIZE; - dictSize = LIZ_DICT_SIZE; - } - LIZ_init (ctxPtr, (const BYTE*)dictionary); - if (dictSize >= HASH_UPDATE_LIMIT) LIZ_Insert (ctxPtr, (const BYTE*)dictionary + (dictSize - (HASH_UPDATE_LIMIT-1))); - ctxPtr->end = (const BYTE*)dictionary + dictSize; - return dictSize; -} - - -static void LIZ_setExternalDict(LIZ_stream_t* ctxPtr, const BYTE* newBlock) -{ - if (ctxPtr->end >= ctxPtr->base + HASH_UPDATE_LIMIT) LIZ_Insert (ctxPtr, ctxPtr->end - (HASH_UPDATE_LIMIT-1)); /* Referencing remaining dictionary content */ - /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ - ctxPtr->lowLimit = ctxPtr->dictLimit; - ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); - ctxPtr->dictBase = ctxPtr->base; - ctxPtr->base = newBlock - ctxPtr->dictLimit; - ctxPtr->end = newBlock; - ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ -} - - -/* dictionary saving */ -int LIZ_saveDict (LIZ_stream_t* LIZ_streamPtr, char* safeBuffer, int dictSize) -{ - LIZ_stream_t* const ctx = (LIZ_stream_t*)LIZ_streamPtr; - int const prefixSize = (int)(ctx->end - (ctx->base + ctx->dictLimit)); -//printf("LIZ_saveDict dictSize=%d prefixSize=%d ctx->dictLimit=%d\n", dictSize, prefixSize, (int)ctx->dictLimit); - if (dictSize > LIZ_DICT_SIZE) dictSize = LIZ_DICT_SIZE; - if (dictSize < 4) dictSize = 0; - if (dictSize > prefixSize) dictSize = prefixSize; - memmove(safeBuffer, ctx->end - dictSize, dictSize); - { U32 const endIndex = (U32)(ctx->end - ctx->base); - ctx->end = (const BYTE*)safeBuffer + dictSize; - ctx->base = ctx->end - endIndex; - ctx->dictLimit = endIndex - dictSize; - ctx->lowLimit = endIndex - dictSize; - if (ctx->nextToUpdate < ctx->dictLimit) ctx->nextToUpdate = ctx->dictLimit; - } -//printf("2LIZ_saveDict dictSize=%d\n", dictSize); - return dictSize; -} - -FORCE_INLINE int LIZ_compress_generic ( - void* ctxvoid, - const char* source, - char* dest, - int inputSize, - int maxOutputSize) -{ - LIZ_stream_t* ctx = (LIZ_stream_t*) ctxvoid; - size_t dictSize = (size_t)(ctx->end - ctx->base) - ctx->dictLimit; - const BYTE* ip = (const BYTE*) source; - BYTE* op = (BYTE*) dest; - BYTE* const oend = op + maxOutputSize; - int res; - - (void)dictSize; - LIZ_LOG_COMPRESS("LIZ_compress_generic source=%p inputSize=%d dest=%p maxOutputSize=%d cLevel=%d dictBase=%p dictSize=%d\n", source, inputSize, dest, maxOutputSize, ctx->compressionLevel, ctx->dictBase, (int)dictSize); - *op++ = (BYTE)ctx->compressionLevel; - maxOutputSize--; // can be lower than 0 - ctx->end += inputSize; - ctx->srcBase = ctx->off24pos = ip; - ctx->destBase = (BYTE*)dest; - - while (inputSize > 0) - { - int inputPart = MIN(LIZ_BLOCK_SIZE, inputSize); - - if (ctx->huffType) LIZ_rescaleFreqs(ctx); - LIZ_initBlock(ctx); - ctx->diffBase = ip; - - switch(ctx->params.parserType) - { - default: - case LIZ_parser_fastSmall: - res = LIZ_compress_fastSmall(ctx, ip, ip+inputPart); break; - case LIZ_parser_fast: - res = LIZ_compress_fast(ctx, ip, ip+inputPart); break; - case LIZ_parser_noChain: - res = LIZ_compress_noChain(ctx, ip, ip+inputPart); break; - case LIZ_parser_hashChain: - res = LIZ_compress_hashChain(ctx, ip, ip+inputPart); break; -#ifndef USE_LZ4_ONLY - case LIZ_parser_fastBig: - res = LIZ_compress_fastBig(ctx, ip, ip+inputPart); break; - case LIZ_parser_priceFast: - res = LIZ_compress_priceFast(ctx, ip, ip+inputPart); break; - case LIZ_parser_lowestPrice: - res = LIZ_compress_lowestPrice(ctx, ip, ip+inputPart); break; - case LIZ_parser_optimalPrice: - case LIZ_parser_optimalPriceBT: - res = LIZ_compress_optimalPrice(ctx, ip, ip+inputPart); break; -#else - case LIZ_parser_priceFast: - case LIZ_parser_lowestPrice: - case LIZ_parser_optimalPrice: - case LIZ_parser_optimalPriceBT: - res = 0; -#endif - } - - LIZ_LOG_COMPRESS("LIZ_compress_generic res=%d inputPart=%d \n", res, inputPart); - if (res <= 0) return res; - - if (LIZ_writeBlock(ctx, ip, inputPart, &op, oend)) goto _output_error; - - ip += inputPart; - inputSize -= inputPart; - LIZ_LOG_COMPRESS("LIZ_compress_generic in=%d out=%d\n", (int)(ip-(const BYTE*)source), (int)(op-(BYTE*)dest)); - } - - LIZ_LOG_COMPRESS("LIZ_compress_generic total=%d\n", (int)(op-(BYTE*)dest)); - return (int)(op-(BYTE*)dest); -_output_error: - LIZ_LOG_COMPRESS("LIZ_compress_generic ERROR\n"); - return 0; -} - - -int LIZ_compress_continue (LIZ_stream_t* ctxPtr, - const char* source, char* dest, - int inputSize, int maxOutputSize) -{ - /* auto-init if forgotten */ - if (ctxPtr->base == NULL) LIZ_init (ctxPtr, (const BYTE*) source); - - /* Check overflow */ - if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) { - size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit; - if (dictSize > LIZ_DICT_SIZE) dictSize = LIZ_DICT_SIZE; - LIZ_loadDict((LIZ_stream_t*)ctxPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize); - } - - /* Check if blocks follow each other */ - if ((const BYTE*)source != ctxPtr->end) - LIZ_setExternalDict(ctxPtr, (const BYTE*)source); - - /* Check overlapping input/dictionary space */ - { const BYTE* sourceEnd = (const BYTE*) source + inputSize; - const BYTE* const dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit; - const BYTE* const dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit; - if ((sourceEnd > dictBegin) && ((const BYTE*)source < dictEnd)) { - if (sourceEnd > dictEnd) sourceEnd = dictEnd; - ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase); - if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit; - } - } - - return LIZ_compress_generic (ctxPtr, source, dest, inputSize, maxOutputSize); -} - - -int LIZ_compress_extState (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel) -{ - LIZ_stream_t* ctx = (LIZ_stream_t*) state; - if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ - - /* initialize stream */ - LIZ_initStream(ctx, compressionLevel); - LIZ_init ((LIZ_stream_t*)state, (const BYTE*)src); - - return LIZ_compress_generic (state, src, dst, srcSize, maxDstSize); -} - - -int LIZ_compress(const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel) -{ - int cSize; - LIZ_stream_t* statePtr = LIZ_createStream(compressionLevel); - - if (!statePtr) return 0; - cSize = LIZ_compress_extState(statePtr, src, dst, srcSize, maxDstSize, compressionLevel); - - LIZ_freeStream(statePtr); - return cSize; -} - - -/************************************** -* Level1 functions -**************************************/ -int LIZ_compress_extState_MinLevel(void* state, const char* source, char* dest, int inputSize, int maxOutputSize) -{ - return LIZ_compress_extState(state, source, dest, inputSize, maxOutputSize, LIZ_MIN_CLEVEL); -} - -int LIZ_compress_MinLevel(const char* source, char* dest, int inputSize, int maxOutputSize) -{ - return LIZ_compress(source, dest, inputSize, maxOutputSize, LIZ_MIN_CLEVEL); -} - -LIZ_stream_t* LIZ_createStream_MinLevel(void) -{ - return LIZ_createStream(LIZ_MIN_CLEVEL); -} - -LIZ_stream_t* LIZ_resetStream_MinLevel(LIZ_stream_t* LIZ_stream) -{ - return LIZ_resetStream (LIZ_stream, LIZ_MIN_CLEVEL); -} diff --git a/C/lizard/liz_lz5_decompress.c b/C/lizard/liz_lz5_decompress.c deleted file mode 100644 index e44fde7c..00000000 --- a/C/lizard/liz_lz5_decompress.c +++ /dev/null @@ -1,375 +0,0 @@ -/* - LZ5 - Fast LZ compression algorithm - Copyright (C) 2011-2016, Yann Collet. - Copyright (C) 2016, Przemyslaw Skibinski - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ5 source repository : https://github.com/inikep/lz5 -*/ - - -/************************************** -* Includes -**************************************/ -//#define LIZ_STATS 1 // 0=simple stats, 1=more, 2=full -#ifdef LIZ_STATS - #include "test/lz5_stats.h" -#endif -#include "liz_compress.h" -#include "liz_decompress.h" -#include "liz_common.h" -#include // printf -#include // intptr_t - - -/*-************************************ -* Local Structures and types -**************************************/ -typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive; -typedef enum { full = 0, partial = 1 } earlyEnd_directive; - -#include "liz_decompress_lz4.h" -#ifndef USE_LZ4_ONLY - #ifdef LIZ_USE_TEST - #include "test/lz5_common_test.h" - #include "test/lz5_decompress_test.h" - #else - #include "liz_decompress_lz5v2.h" - #endif -#endif -#include "huf.h" - - -/*-***************************** -* Decompression functions -*******************************/ - -FORCE_INLINE size_t LIZ_readStream(int flag, const BYTE** ip, const BYTE* const iend, BYTE* op, BYTE* const oend, const BYTE** streamPtr, const BYTE** streamEnd, int streamFlag) -{ - if (!flag) { - if (*ip > iend - 3) return 0; - *streamPtr = *ip + 3; - *streamEnd = *streamPtr + MEM_readLE24(*ip); - if (*streamEnd < *streamPtr) return 0; - *ip = *streamEnd; -#ifdef LIZ_STATS - uncompr_stream[streamFlag] += *streamEnd-*streamPtr; -#else - (void)streamFlag; -#endif - return 1; - } else { -#ifndef LIZ_NO_HUFFMAN - size_t res, streamLen, comprStreamLen; - - if (*ip > iend - 6) return 0; - streamLen = MEM_readLE24(*ip); - comprStreamLen = MEM_readLE24(*ip + 3); - - // printf("LIZ_readStream ip=%p iout=%p iend=%p streamLen=%d comprStreamLen=%d\n", *ip, *ip + 6 + comprStreamLen, iend, (int)streamLen, (int)comprStreamLen); - - if ((op > oend - streamLen) || (*ip + comprStreamLen > iend - 6)) return 0; - res = LIZHUF_decompress(op, streamLen, *ip + 6, comprStreamLen); - if (LIZHUF_isError(res) || (res != streamLen)) return 0; - - *ip += comprStreamLen + 6; - *streamPtr = op; - *streamEnd = *streamPtr + streamLen; -#ifdef LIZ_STATS - compr_stream[streamFlag] += comprStreamLen + 6; - decompr_stream[streamFlag] += *streamEnd-*streamPtr; -#endif - return 1; -#else - fprintf(stderr, "compiled with LIZ_NO_HUFFMAN\n"); - (void)op; (void)oend; - return 0; -#endif - } -} - - -FORCE_INLINE int LIZ_decompress_generic( - const char* source, - char* const dest, - int inputSize, - int outputSize, /* this value is the max size of Output Buffer. */ - int partialDecoding, /* full, partial */ - int targetOutputSize, /* only used if partialDecoding==partial */ - int dict, /* noDict, withPrefix64k, usingExtDict */ - const BYTE* const lowPrefix, /* == dest if dict == noDict */ - const BYTE* const dictStart, /* only if dict==usingExtDict */ - const size_t dictSize /* note : = 0 if noDict */ - ) -{ - /* Local Variables */ - const BYTE* ip = (const BYTE*) source, *istart = (const BYTE*) source; - const BYTE* const iend = ip + inputSize; - BYTE* op = (BYTE*) dest; - BYTE* const oend = op + outputSize; - BYTE* oexit = op + targetOutputSize; - LIZ_parameters params; - LIZ_dstream_t ctx; - BYTE* decompFlagsBase, *decompOff24Base, *decompOff16Base, *decompLiteralsBase = NULL; - int res, compressionLevel; - - if (inputSize < 1) { LIZ_LOG_DECOMPRESS("inputSize=%d outputSize=%d targetOutputSize=%d partialDecoding=%d\n", inputSize, outputSize, targetOutputSize, partialDecoding); return 0; } - - compressionLevel = *ip++; - - if (compressionLevel < LIZ_MIN_CLEVEL || compressionLevel > LIZ_MAX_CLEVEL) { - LIZ_LOG_DECOMPRESS("ERROR LIZ_decompress_generic inputSize=%d compressionLevel=%d\n", inputSize, compressionLevel); - return -1; - } - - LIZ_LOG_DECOMPRESS("LIZ_decompress_generic ip=%p inputSize=%d targetOutputSize=%d dest=%p outputSize=%d cLevel=%d dict=%d dictSize=%d dictStart=%p partialDecoding=%d\n", ip, inputSize, targetOutputSize, dest, outputSize, compressionLevel, dict, (int)dictSize, dictStart, partialDecoding); - - decompLiteralsBase = (BYTE*)malloc(4*LIZ_LIZHUF_BLOCK_SIZE); - if (!decompLiteralsBase) return -1; - decompFlagsBase = decompLiteralsBase + LIZ_LIZHUF_BLOCK_SIZE; - decompOff24Base = decompFlagsBase + LIZ_LIZHUF_BLOCK_SIZE; - decompOff16Base = decompOff24Base + LIZ_LIZHUF_BLOCK_SIZE; - -#ifdef LIZ_STATS - init_stats(); -#endif - (void)istart; - - while (ip < iend) - { - res = *ip++; - if (res == LIZ_FLAG_UNCOMPRESSED) /* uncompressed */ - { - uint32_t length; - if (ip > iend - 3) { LIZ_LOG_DECOMPRESS("UNCOMPRESSED ip[%p] > iend[%p] - 3\n", ip, iend); goto _output_error; } - length = MEM_readLE24(ip); - ip += 3; - // printf("%d: total=%d block=%d UNCOMPRESSED op=%p oexit=%p oend=%p\n", (int)(op-(BYTE*)dest) ,(int)(ip-istart), length, op, oexit, oend); - if (ip + length > iend || op + length > oend) { LIZ_LOG_DECOMPRESS("UNCOMPRESSED ip[%p]+length[%d] > iend[%p]\n", ip, length, iend); goto _output_error; } - memcpy(op, ip, length); - op += length; - ip += length; - if ((partialDecoding) && (op >= oexit)) break; -#ifdef LIZ_STATS - uncompr_stream[LIZ_STREAM_UNCOMPRESSED] += length; -#endif - continue; - } - - if (res&LIZ_FLAG_LEN) { - LIZ_LOG_DECOMPRESS("res=%d\n", res); goto _output_error; - } - - if (ip > iend - 5*3) goto _output_error; - ctx.lenPtr = (const BYTE*)ip + 3; - ctx.lenEnd = ctx.lenPtr + MEM_readLE24(ip); - if (ctx.lenEnd < ctx.lenPtr || (ctx.lenEnd > iend - 3)) goto _output_error; -#ifdef LIZ_STATS - uncompr_stream[LIZ_STREAM_LEN] += ctx.lenEnd-ctx.lenPtr + 3; -#endif - ip = ctx.lenEnd; - - { size_t streamLen; -#ifdef LIZ_USE_LOGS - const BYTE* ipos; - size_t comprFlagsLen, comprLiteralsLen, total; -#endif - streamLen = LIZ_readStream(res&LIZ_FLAG_OFFSET16, &ip, iend, decompOff16Base, decompOff16Base + LIZ_LIZHUF_BLOCK_SIZE, &ctx.offset16Ptr, &ctx.offset16End, LIZ_STREAM_OFFSET16); - if (streamLen == 0) goto _output_error; - - streamLen = LIZ_readStream(res&LIZ_FLAG_OFFSET24, &ip, iend, decompOff24Base, decompOff24Base + LIZ_LIZHUF_BLOCK_SIZE, &ctx.offset24Ptr, &ctx.offset24End, LIZ_STREAM_OFFSET24); - if (streamLen == 0) goto _output_error; - -#ifdef LIZ_USE_LOGS - ipos = ip; - streamLen = LIZ_readStream(res&LIZ_FLAG_FLAGS, &ip, iend, decompFlagsBase, decompFlagsBase + LIZ_LIZHUF_BLOCK_SIZE, &ctx.flagsPtr, &ctx.flagsEnd, LIZ_STREAM_FLAGS); - if (streamLen == 0) goto _output_error; - streamLen = (size_t)(ctx.flagsEnd-ctx.flagsPtr); - comprFlagsLen = ((size_t)(ip - ipos) + 3 >= streamLen) ? 0 : (size_t)(ip - ipos); - ipos = ip; -#else - streamLen = LIZ_readStream(res&LIZ_FLAG_FLAGS, &ip, iend, decompFlagsBase, decompFlagsBase + LIZ_LIZHUF_BLOCK_SIZE, &ctx.flagsPtr, &ctx.flagsEnd, LIZ_STREAM_FLAGS); - if (streamLen == 0) goto _output_error; -#endif - - streamLen = LIZ_readStream(res&LIZ_FLAG_LITERALS, &ip, iend, decompLiteralsBase, decompLiteralsBase + LIZ_LIZHUF_BLOCK_SIZE, &ctx.literalsPtr, &ctx.literalsEnd, LIZ_STREAM_LITERALS); - if (streamLen == 0) goto _output_error; -#ifdef LIZ_USE_LOGS - streamLen = (size_t)(ctx.literalsEnd-ctx.literalsPtr); - comprLiteralsLen = ((size_t)(ip - ipos) + 3 >= streamLen) ? 0 : (size_t)(ip - ipos); - total = (size_t)(ip-(ctx.lenEnd-1)); -#endif - - if (ip > iend) goto _output_error; - - LIZ_LOG_DECOMPRESS("%d: total=%d block=%d flagsLen=%d(HUF=%d) literalsLen=%d(HUF=%d) offset16Len=%d offset24Len=%d lengthsLen=%d \n", (int)(op-(BYTE*)dest) ,(int)(ip-istart), (int)total, - (int)(ctx.flagsEnd-ctx.flagsPtr), (int)comprFlagsLen, (int)(ctx.literalsEnd-ctx.literalsPtr), (int)comprLiteralsLen, - (int)(ctx.offset16End-ctx.offset16Ptr), (int)(ctx.offset24End-ctx.offset24Ptr), (int)(ctx.lenEnd-ctx.lenPtr)); - } - - ctx.last_off = -LIZ_INIT_LAST_OFFSET; - params = LIZ_defaultParameters[compressionLevel - LIZ_MIN_CLEVEL]; - if (params.decompressType == LIZ_coderwords_LZ4) - res = LIZ_decompress_LZ4(&ctx, op, outputSize, partialDecoding, targetOutputSize, dict, lowPrefix, dictStart, dictSize, compressionLevel); - else -#ifdef USE_LZ4_ONLY - res = LIZ_decompress_LZ4(&ctx, op, outputSize, partialDecoding, targetOutputSize, dict, lowPrefix, dictStart, dictSize, compressionLevel); -#else - res = LIZ_decompress_LZ5v2(&ctx, op, outputSize, partialDecoding, targetOutputSize, dict, lowPrefix, dictStart, dictSize, compressionLevel); -#endif - LIZ_LOG_DECOMPRESS("LIZ_decompress_generic res=%d inputSize=%d\n", res, (int)(ctx.literalsEnd-ctx.lenEnd)); - - if (res <= 0) { free(decompLiteralsBase); return res; } - - op += res; - outputSize -= res; - if ((partialDecoding) && (op >= oexit)) break; - } - -#ifdef LIZ_STATS - print_stats(); -#endif - - LIZ_LOG_DECOMPRESS("LIZ_decompress_generic total=%d\n", (int)(op-(BYTE*)dest)); - free(decompLiteralsBase); - return (int)(op-(BYTE*)dest); - -_output_error: - LIZ_LOG_DECOMPRESS("LIZ_decompress_generic ERROR\n"); - free(decompLiteralsBase); - return -1; -} - - -int LIZ_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) -{ - return LIZ_decompress_generic(source, dest, compressedSize, maxDecompressedSize, full, 0, noDict, (BYTE*)dest, NULL, 0); -} - -int LIZ_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize) -{ - return LIZ_decompress_generic(source, dest, compressedSize, maxDecompressedSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0); -} - - -/*===== streaming decompression functions =====*/ - - -/* - * If you prefer dynamic allocation methods, - * LIZ_createStreamDecode() - * provides a pointer (void*) towards an initialized LIZ_streamDecode_t structure. - */ -LIZ_streamDecode_t* LIZ_createStreamDecode(void) -{ - LIZ_streamDecode_t* lz5s = (LIZ_streamDecode_t*) ALLOCATOR(1, sizeof(LIZ_streamDecode_t)); - (void)LIZ_count; /* unused function 'LIZ_count' */ - return lz5s; -} - -int LIZ_freeStreamDecode (LIZ_streamDecode_t* LIZ_stream) -{ - FREEMEM(LIZ_stream); - return 0; -} - -/*! - * LIZ_setStreamDecode() : - * Use this function to instruct where to find the dictionary. - * This function is not necessary if previous data is still available where it was decoded. - * Loading a size of 0 is allowed (same effect as no dictionary). - * Return : 1 if OK, 0 if error - */ -int LIZ_setStreamDecode (LIZ_streamDecode_t* LIZ_streamDecode, const char* dictionary, int dictSize) -{ - LIZ_streamDecode_t* lz5sd = (LIZ_streamDecode_t*) LIZ_streamDecode; - lz5sd->prefixSize = (size_t) dictSize; - lz5sd->prefixEnd = (const BYTE*) dictionary + dictSize; - lz5sd->externalDict = NULL; - lz5sd->extDictSize = 0; - return 1; -} - -/* -*_continue() : - These decoding functions allow decompression of multiple blocks in "streaming" mode. - Previously decoded blocks must still be available at the memory position where they were decoded. - If it's not possible, save the relevant part of decoded data into a safe buffer, - and indicate where it stands using LIZ_setStreamDecode() -*/ -int LIZ_decompress_safe_continue (LIZ_streamDecode_t* LIZ_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) -{ - LIZ_streamDecode_t* lz5sd = (LIZ_streamDecode_t*) LIZ_streamDecode; - int result; - - if (lz5sd->prefixEnd == (BYTE*)dest) { - result = LIZ_decompress_generic(source, dest, compressedSize, maxOutputSize, - full, 0, usingExtDict, lz5sd->prefixEnd - lz5sd->prefixSize, lz5sd->externalDict, lz5sd->extDictSize); - if (result <= 0) return result; - lz5sd->prefixSize += result; - lz5sd->prefixEnd += result; - } else { - lz5sd->extDictSize = lz5sd->prefixSize; - lz5sd->externalDict = lz5sd->prefixEnd - lz5sd->extDictSize; - result = LIZ_decompress_generic(source, dest, compressedSize, maxOutputSize, - full, 0, usingExtDict, (BYTE*)dest, lz5sd->externalDict, lz5sd->extDictSize); - if (result <= 0) return result; - lz5sd->prefixSize = result; - lz5sd->prefixEnd = (BYTE*)dest + result; - } - - return result; -} - - -/* -Advanced decoding functions : -*_usingDict() : - These decoding functions work the same as "_continue" ones, - the dictionary must be explicitly provided within parameters -*/ - -int LIZ_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) -{ - if (dictSize==0) - return LIZ_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, noDict, (BYTE*)dest, NULL, 0); - if (dictStart+dictSize == dest) - { - if (dictSize >= (int)(LIZ_DICT_SIZE - 1)) - return LIZ_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, withPrefix64k, (BYTE*)dest-LIZ_DICT_SIZE, NULL, 0); - return LIZ_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); - } - return LIZ_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); -} - -/* debug function */ -int LIZ_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) -{ - return LIZ_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); -} - diff --git a/C/lizard/liz_parser_fast.h b/C/lizard/liz_parser_fast.h deleted file mode 100644 index a4201242..00000000 --- a/C/lizard/liz_parser_fast.h +++ /dev/null @@ -1,200 +0,0 @@ -#define LIZ_FAST_MIN_OFFSET 8 -#define LIZ_FAST_LONGOFF_MM 0 /* not used with offsets > 1<<16 */ - -/************************************** -* Hash Functions -**************************************/ -static size_t LIZ_hashPosition(const void* p) -{ - if (MEM_64bits()) - return LIZ_hash5Ptr(p, LIZ_HASHLOG_LZ4); - return LIZ_hash4Ptr(p, LIZ_HASHLOG_LZ4); -} - -static void LIZ_putPositionOnHash(const BYTE* p, size_t h, U32* hashTable, const BYTE* srcBase) -{ - hashTable[h] = (U32)(p-srcBase); -} - -static void LIZ_putPosition(const BYTE* p, U32* hashTable, const BYTE* srcBase) -{ - size_t const h = LIZ_hashPosition(p); - LIZ_putPositionOnHash(p, h, hashTable, srcBase); -} - -static U32 LIZ_getPositionOnHash(size_t h, U32* hashTable) -{ - return hashTable[h]; -} - -static U32 LIZ_getPosition(const BYTE* p, U32* hashTable) -{ - size_t const h = LIZ_hashPosition(p); - return LIZ_getPositionOnHash(h, hashTable); -} - - -static const U32 LIZ_skipTrigger = 6; /* Increase this value ==> compression run slower on incompressible data */ -static const U32 LIZ_minLength = (MFLIMIT+1); - - -FORCE_INLINE int LIZ_compress_fast( - LIZ_stream_t* const ctx, - const BYTE* ip, - const BYTE* const iend) -{ - const U32 acceleration = 1; - const BYTE* base = ctx->base; - const U32 lowLimit = ctx->lowLimit; - const U32 dictLimit = ctx->dictLimit; - const BYTE* const lowPrefixPtr = base + dictLimit; - const BYTE* const dictBase = ctx->dictBase; - const BYTE* const dictEnd = dictBase + dictLimit; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = iend - LASTLITERALS; - const BYTE* anchor = ip; - - size_t forwardH, matchIndex; - const U32 maxDistance = (1 << ctx->params.windowLog) - 1; - - // fprintf(stderr, "base=%p LIZ_stream_t=%d inputSize=%d maxOutputSize=%d\n", base, sizeof(LIZ_stream_t), inputSize, maxOutputSize); - // fprintf(stderr, "ip=%d base=%p lowPrefixPtr=%p dictBase=%d lowLimit=%p op=%p\n", ip, base, lowPrefixPtr, lowLimit, dictBase, op); - - /* Init conditions */ - if ((U32)(iend-ip) > (U32)LIZ_MAX_INPUT_SIZE) goto _output_error; /* Unsupported inputSize, too large (or negative) */ - - if ((U32)(iend-ip) < LIZ_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ - - /* First Byte */ - LIZ_putPosition(ip, ctx->hashTable, base); - ip++; forwardH = LIZ_hashPosition(ip); - - /* Main Loop */ - for ( ; ; ) { - const BYTE* match; - // BYTE* token; - size_t matchLength; - - /* Find a match */ - { const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = acceleration << LIZ_skipTrigger; - while (1) { - size_t const h = forwardH; - ip = forwardIp; - forwardIp += step; - step = (searchMatchNb++ >> LIZ_skipTrigger); - - if (unlikely(forwardIp > mflimit)) goto _last_literals; - - matchIndex = LIZ_getPositionOnHash(h, ctx->hashTable); - forwardH = LIZ_hashPosition(forwardIp); - LIZ_putPositionOnHash(ip, h, ctx->hashTable, base); - - if ((matchIndex < lowLimit) || (base + matchIndex + maxDistance < ip)) continue; - - if (matchIndex >= dictLimit) { - match = base + matchIndex; -#if LIZ_FAST_MIN_OFFSET > 0 - if ((U32)(ip - match) >= LIZ_FAST_MIN_OFFSET) -#endif - if (MEM_read32(match) == MEM_read32(ip)) - { - int back = 0; - matchLength = LIZ_count(ip+MINMATCH, match+MINMATCH, matchlimit); - - while ((ip+back > anchor) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; - matchLength -= back; -#if LIZ_FAST_LONGOFF_MM > 0 - if ((matchLength >= LIZ_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZ_MAX_16BIT_OFFSET)) -#endif - { - ip += back; - match += back; - break; - } - } - } else { - match = dictBase + matchIndex; -#if LIZ_FAST_MIN_OFFSET > 0 - if ((U32)(ip - (base + matchIndex)) >= LIZ_FAST_MIN_OFFSET) -#endif - if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ - if (MEM_read32(match) == MEM_read32(ip)) { - const U32 newLowLimit = (lowLimit + maxDistance >= (U32)(ip-base)) ? lowLimit : (U32)(ip - base) - maxDistance; - int back = 0; - matchLength = LIZ_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr); - - while ((ip+back > anchor) && (matchIndex+back > newLowLimit) && (ip[back-1] == match[back-1])) back--; - matchLength -= back; - match = base + matchIndex + back; -#if LIZ_FAST_LONGOFF_MM > 0 - if ((matchLength >= LIZ_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZ_MAX_16BIT_OFFSET)) -#endif - { - ip += back; - break; - } - } - } - } // while (1) - } - -_next_match: - if (LIZ_encodeSequence_LZ4(ctx, &ip, &anchor, matchLength+MINMATCH, match)) goto _output_error; - - /* Test end of chunk */ - if (ip > mflimit) break; - - /* Fill table */ - LIZ_putPosition(ip-2, ctx->hashTable, base); - - /* Test next position */ - matchIndex = LIZ_getPosition(ip, ctx->hashTable); - LIZ_putPosition(ip, ctx->hashTable, base); - if (matchIndex >= lowLimit && (base + matchIndex + maxDistance >= ip)) - { - if (matchIndex >= dictLimit) { - match = base + matchIndex; -#if LIZ_FAST_MIN_OFFSET > 0 - if ((U32)(ip - match) >= LIZ_FAST_MIN_OFFSET) -#endif - if (MEM_read32(match) == MEM_read32(ip)) - { - matchLength = LIZ_count(ip+MINMATCH, match+MINMATCH, matchlimit); -#if LIZ_FAST_LONGOFF_MM > 0 - if ((matchLength >= LIZ_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZ_MAX_16BIT_OFFSET)) -#endif - goto _next_match; - } - } else { - match = dictBase + matchIndex; -#if LIZ_FAST_MIN_OFFSET > 0 - if ((U32)(ip - (base + matchIndex)) >= LIZ_FAST_MIN_OFFSET) -#endif - if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ - if (MEM_read32(match) == MEM_read32(ip)) { - matchLength = LIZ_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr); - match = base + matchIndex; -#if LIZ_FAST_LONGOFF_MM > 0 - if ((matchLength >= LIZ_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZ_MAX_16BIT_OFFSET)) -#endif - goto _next_match; - } - } - } - - /* Prepare next loop */ - forwardH = LIZ_hashPosition(++ip); - } - -_last_literals: - /* Encode Last Literals */ - ip = iend; - if (LIZ_encodeLastLiterals_LZ4(ctx, &ip, &anchor)) goto _output_error; - - /* End */ - return 1; -_output_error: - return 0; -} diff --git a/C/lizard/liz_parser_fastbig.h b/C/lizard/liz_parser_fastbig.h deleted file mode 100644 index 9e797f8b..00000000 --- a/C/lizard/liz_parser_fastbig.h +++ /dev/null @@ -1,179 +0,0 @@ -#define LIZ_FASTBIG_LONGOFF_MM MM_LONGOFF - -/************************************** -* Hash Functions -**************************************/ -static size_t LIZ_hashPositionHLog(const void* p, int hashLog) -{ - if (MEM_64bits()) - return LIZ_hash5Ptr(p, hashLog); - return LIZ_hash4Ptr(p, hashLog); -} - -static void LIZ_putPositionOnHashHLog(const BYTE* p, size_t h, U32* hashTable, const BYTE* srcBase) -{ - hashTable[h] = (U32)(p-srcBase); -} - -static void LIZ_putPositionHLog(const BYTE* p, U32* hashTable, const BYTE* srcBase, int hashLog) -{ - size_t const h = LIZ_hashPositionHLog(p, hashLog); - LIZ_putPositionOnHashHLog(p, h, hashTable, srcBase); -} - -static U32 LIZ_getPositionOnHashHLog(size_t h, U32* hashTable) -{ - return hashTable[h]; -} - -static U32 LIZ_getPositionHLog(const BYTE* p, U32* hashTable, int hashLog) -{ - size_t const h = LIZ_hashPositionHLog(p, hashLog); - return LIZ_getPositionOnHashHLog(h, hashTable); -} - -FORCE_INLINE int LIZ_compress_fastBig( - LIZ_stream_t* const ctx, - const BYTE* ip, - const BYTE* const iend) -{ - const U32 acceleration = 1; - const BYTE* base = ctx->base; - const U32 lowLimit = ctx->lowLimit; - const U32 dictLimit = ctx->dictLimit; - const BYTE* const lowPrefixPtr = base + dictLimit; - const BYTE* const dictBase = ctx->dictBase; - const BYTE* const dictEnd = dictBase + dictLimit; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = iend - LASTLITERALS; - const BYTE* anchor = ip; - - size_t forwardH, matchIndex; - const int hashLog = ctx->params.hashLog; - const U32 maxDistance = (1 << ctx->params.windowLog) - 1; - - // fprintf(stderr, "base=%p LIZ_stream_t=%d inputSize=%d maxOutputSize=%d\n", base, sizeof(LIZ_stream_t), inputSize, maxOutputSize); - // fprintf(stderr, "ip=%d base=%p lowPrefixPtr=%p dictBase=%d lowLimit=%p op=%p\n", ip, base, lowPrefixPtr, lowLimit, dictBase, op); - - /* Init conditions */ - if ((U32)(iend-ip) > (U32)LIZ_MAX_INPUT_SIZE) goto _output_error; /* Unsupported inputSize, too large (or negative) */ - - if ((U32)(iend-ip) < LIZ_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ - - /* First Byte */ - LIZ_putPositionHLog(ip, ctx->hashTable, base, hashLog); - ip++; forwardH = LIZ_hashPositionHLog(ip, hashLog); - - /* Main Loop */ - for ( ; ; ) { - const BYTE* match; - // BYTE* token; - size_t matchLength; - - /* Find a match */ - { const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = acceleration << LIZ_skipTrigger; - while (1) { - size_t const h = forwardH; - ip = forwardIp; - forwardIp += step; - step = (searchMatchNb++ >> LIZ_skipTrigger); - - if (unlikely(forwardIp > mflimit)) goto _last_literals; - - matchIndex = LIZ_getPositionOnHashHLog(h, ctx->hashTable); - forwardH = LIZ_hashPositionHLog(forwardIp, hashLog); - LIZ_putPositionOnHashHLog(ip, h, ctx->hashTable, base); - - if ((matchIndex < lowLimit) || (base + matchIndex + maxDistance < ip)) continue; - - if (matchIndex >= dictLimit) { - match = base + matchIndex; - if ((U32)(ip - match) >= LIZ_FAST_MIN_OFFSET) - if (MEM_read32(match) == MEM_read32(ip)) - { - int back = 0; - matchLength = LIZ_count(ip+MINMATCH, match+MINMATCH, matchlimit); - - while ((ip+back > anchor) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; - matchLength -= back; - if ((matchLength >= LIZ_FASTBIG_LONGOFF_MM) || ((U32)(ip - match) < LIZ_MAX_16BIT_OFFSET)) - { - ip += back; - match += back; - break; - } - } - } else { - match = dictBase + matchIndex; - if ((U32)(ip - (base + matchIndex)) >= LIZ_FAST_MIN_OFFSET) - if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ - if (MEM_read32(match) == MEM_read32(ip)) { - const U32 newLowLimit = (lowLimit + maxDistance >= (U32)(ip-base)) ? lowLimit : (U32)(ip - base) - maxDistance; - int back = 0; - matchLength = LIZ_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr); - - while ((ip+back > anchor) && (matchIndex+back > newLowLimit) && (ip[back-1] == match[back-1])) back--; - matchLength -= back; - match = base + matchIndex + back; - if ((matchLength >= LIZ_FASTBIG_LONGOFF_MM) || ((U32)(ip - match) < LIZ_MAX_16BIT_OFFSET)) - { - ip += back; - break; - } - } - } - } // while (1) - } - -_next_match: - if (LIZ_encodeSequence_LZ5v2(ctx, &ip, &anchor, matchLength+MINMATCH, match)) goto _output_error; - - /* Test end of chunk */ - if (ip > mflimit) break; - - /* Fill table */ - LIZ_putPositionHLog(ip-2, ctx->hashTable, base, hashLog); - - /* Test next position */ - matchIndex = LIZ_getPositionHLog(ip, ctx->hashTable, hashLog); - LIZ_putPositionHLog(ip, ctx->hashTable, base, hashLog); - if (matchIndex >= lowLimit && (base + matchIndex + maxDistance >= ip)) - { - if (matchIndex >= dictLimit) { - match = base + matchIndex; - if ((U32)(ip - match) >= LIZ_FAST_MIN_OFFSET) - if (MEM_read32(match) == MEM_read32(ip)) - { - matchLength = LIZ_count(ip+MINMATCH, match+MINMATCH, matchlimit); - if ((matchLength >= LIZ_FASTBIG_LONGOFF_MM) || ((U32)(ip - match) < LIZ_MAX_16BIT_OFFSET)) - goto _next_match; - } - } else { - match = dictBase + matchIndex; - if ((U32)(ip - (base + matchIndex)) >= LIZ_FAST_MIN_OFFSET) - if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ - if (MEM_read32(match) == MEM_read32(ip)) { - matchLength = LIZ_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr); - match = base + matchIndex; - if ((matchLength >= LIZ_FASTBIG_LONGOFF_MM) || ((U32)(ip - match) < LIZ_MAX_16BIT_OFFSET)) - goto _next_match; - } - } - } - - /* Prepare next loop */ - forwardH = LIZ_hashPositionHLog(++ip, hashLog); - } - -_last_literals: - /* Encode Last Literals */ - ip = iend; - if (LIZ_encodeLastLiterals_LZ5v2(ctx, &ip, &anchor)) goto _output_error; - - /* End */ - return 1; -_output_error: - return 0; -} diff --git a/C/lizard/liz_parser_fastsmall.h b/C/lizard/liz_parser_fastsmall.h deleted file mode 100644 index 04772cbb..00000000 --- a/C/lizard/liz_parser_fastsmall.h +++ /dev/null @@ -1,193 +0,0 @@ -/************************************** -* Hash Functions -**************************************/ -static size_t LIZ_hashPositionSmall(const void* p) -{ - if (MEM_64bits()) - return LIZ_hash5Ptr(p, LIZ_HASHLOG_LZ4SM); - return LIZ_hash4Ptr(p, LIZ_HASHLOG_LZ4SM); -} - -static void LIZ_putPositionOnHashSmall(const BYTE* p, size_t h, U32* hashTable, const BYTE* srcBase) -{ - hashTable[h] = (U32)(p-srcBase); -} - -static void LIZ_putPositionSmall(const BYTE* p, U32* hashTable, const BYTE* srcBase) -{ - size_t const h = LIZ_hashPositionSmall(p); - LIZ_putPositionOnHashSmall(p, h, hashTable, srcBase); -} - -static U32 LIZ_getPositionOnHashSmall(size_t h, U32* hashTable) -{ - return hashTable[h]; -} - -static U32 LIZ_getPositionSmall(const BYTE* p, U32* hashTable) -{ - size_t const h = LIZ_hashPositionSmall(p); - return LIZ_getPositionOnHashSmall(h, hashTable); -} - - -FORCE_INLINE int LIZ_compress_fastSmall( - LIZ_stream_t* const ctx, - const BYTE* ip, - const BYTE* const iend) -{ - const U32 acceleration = 1; - const BYTE* base = ctx->base; - const U32 lowLimit = ctx->lowLimit; - const U32 dictLimit = ctx->dictLimit; - const BYTE* const lowPrefixPtr = base + dictLimit; - const BYTE* const dictBase = ctx->dictBase; - const BYTE* const dictEnd = dictBase + dictLimit; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = iend - LASTLITERALS; - const BYTE* anchor = ip; - - size_t forwardH, matchIndex; - const U32 maxDistance = (1 << ctx->params.windowLog) - 1; - - // fprintf(stderr, "base=%p LIZ_stream_t=%d inputSize=%d maxOutputSize=%d\n", base, sizeof(LIZ_stream_t), inputSize, maxOutputSize); - // fprintf(stderr, "ip=%d base=%p lowPrefixPtr=%p dictBase=%d lowLimit=%p op=%p\n", ip, base, lowPrefixPtr, lowLimit, dictBase, op); - - /* Init conditions */ - if ((U32)(iend-ip) > (U32)LIZ_MAX_INPUT_SIZE) goto _output_error; /* Unsupported inputSize, too large (or negative) */ - - if ((U32)(iend-ip) < LIZ_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ - - /* First Byte */ - LIZ_putPositionSmall(ip, ctx->hashTable, base); - ip++; forwardH = LIZ_hashPositionSmall(ip); - - /* Main Loop */ - for ( ; ; ) { - const BYTE* match; - // BYTE* token; - size_t matchLength; - - /* Find a match */ - { const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = acceleration << LIZ_skipTrigger; - while (1) { - size_t const h = forwardH; - ip = forwardIp; - forwardIp += step; - step = (searchMatchNb++ >> LIZ_skipTrigger); - - if (unlikely(forwardIp > mflimit)) goto _last_literals; - - matchIndex = LIZ_getPositionOnHashSmall(h, ctx->hashTable); - forwardH = LIZ_hashPositionSmall(forwardIp); - LIZ_putPositionOnHashSmall(ip, h, ctx->hashTable, base); - - if ((matchIndex < lowLimit) || (base + matchIndex + maxDistance < ip)) continue; - - if (matchIndex >= dictLimit) { - match = base + matchIndex; -#if LIZ_FAST_MIN_OFFSET > 0 - if ((U32)(ip - match) >= LIZ_FAST_MIN_OFFSET) -#endif - if (MEM_read32(match) == MEM_read32(ip)) - { - int back = 0; - matchLength = LIZ_count(ip+MINMATCH, match+MINMATCH, matchlimit); - - while ((ip+back > anchor) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; - matchLength -= back; -#if LIZ_FAST_LONGOFF_MM > 0 - if ((matchLength >= LIZ_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZ_MAX_16BIT_OFFSET)) -#endif - { - ip += back; - match += back; - break; - } - } - } else { - match = dictBase + matchIndex; -#if LIZ_FAST_MIN_OFFSET > 0 - if ((U32)(ip - (base + matchIndex)) >= LIZ_FAST_MIN_OFFSET) -#endif - if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ - if (MEM_read32(match) == MEM_read32(ip)) { - const U32 newLowLimit = (lowLimit + maxDistance >= (U32)(ip-base)) ? lowLimit : (U32)(ip - base) - maxDistance; - int back = 0; - matchLength = LIZ_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr); - - while ((ip+back > anchor) && (matchIndex+back > newLowLimit) && (ip[back-1] == match[back-1])) back--; - matchLength -= back; - match = base + matchIndex + back; -#if LIZ_FAST_LONGOFF_MM > 0 - if ((matchLength >= LIZ_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZ_MAX_16BIT_OFFSET)) -#endif - { - ip += back; - break; - } - } - } - } // while (1) - } - -_next_match: - if (LIZ_encodeSequence_LZ4(ctx, &ip, &anchor, matchLength+MINMATCH, match)) goto _output_error; - - /* Test end of chunk */ - if (ip > mflimit) break; - - /* Fill table */ - LIZ_putPositionSmall(ip-2, ctx->hashTable, base); - - /* Test next position */ - matchIndex = LIZ_getPositionSmall(ip, ctx->hashTable); - LIZ_putPositionSmall(ip, ctx->hashTable, base); - if (matchIndex >= lowLimit && (base + matchIndex + maxDistance >= ip)) - { - if (matchIndex >= dictLimit) { - match = base + matchIndex; -#if LIZ_FAST_MIN_OFFSET > 0 - if ((U32)(ip - match) >= LIZ_FAST_MIN_OFFSET) -#endif - if (MEM_read32(match) == MEM_read32(ip)) - { - matchLength = LIZ_count(ip+MINMATCH, match+MINMATCH, matchlimit); -#if LIZ_FAST_LONGOFF_MM > 0 - if ((matchLength >= LIZ_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZ_MAX_16BIT_OFFSET)) -#endif - goto _next_match; - } - } else { - match = dictBase + matchIndex; -#if LIZ_FAST_MIN_OFFSET > 0 - if ((U32)(ip - (base + matchIndex)) >= LIZ_FAST_MIN_OFFSET) -#endif - if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ - if (MEM_read32(match) == MEM_read32(ip)) { - matchLength = LIZ_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr); - match = base + matchIndex; -#if LIZ_FAST_LONGOFF_MM > 0 - if ((matchLength >= LIZ_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZ_MAX_16BIT_OFFSET)) -#endif - goto _next_match; - } - } - } - - /* Prepare next loop */ - forwardH = LIZ_hashPositionSmall(++ip); - } - -_last_literals: - /* Encode Last Literals */ - ip = iend; - if (LIZ_encodeLastLiterals_LZ4(ctx, &ip, &anchor)) goto _output_error; - - /* End */ - return 1; -_output_error: - return 0; -} diff --git a/C/lizard/lizard_common.h b/C/lizard/lizard_common.h new file mode 100644 index 00000000..429abc2c --- /dev/null +++ b/C/lizard/lizard_common.h @@ -0,0 +1,497 @@ +/* + Lizard - Fast LZ compression algorithm + Copyright (C) 2011-2015, Yann Collet + Copyright (C) 2016-2017, Przemyslaw Skibinski + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Lizard source repository : https://github.com/inikep/lizard +*/ + +#ifndef LIZARD_COMMON_H_2983 +#define LIZARD_COMMON_H_2983 + +#if defined (__cplusplus) +extern "C" { +#endif + + +/*-************************************ +* Memory routines +**************************************/ +#include /* malloc, calloc, free */ +#include /* memset, memcpy */ +#include /* intptr_t */ +#include "../zstd/mem.h" +#include "lizard_compress.h" /* LIZARD_GCC_VERSION */ + +//#define LIZARD_USE_LOGS +#define LIZARD_LOG_COMPRESS(...) //printf(__VA_ARGS__) +#define LIZARD_LOG_DECOMPRESS(...) //printf(__VA_ARGS__) + +#define LIZARD_LOG_COMPRESS_LZ4(...) //printf(__VA_ARGS__) +#define COMPLOG_CODEWORDS_LZ4(...) //printf(__VA_ARGS__) +#define LIZARD_LOG_DECOMPRESS_LZ4(...) //printf(__VA_ARGS__) +#define DECOMPLOG_CODEWORDS_LZ4(...) //printf(__VA_ARGS__) + +#define LIZARD_LOG_COMPRESS_LIZv1(...) //printf(__VA_ARGS__) +#define COMPLOG_CODEWORDS_LIZv1(...) //printf(__VA_ARGS__) +#define LIZARD_LOG_DECOMPRESS_LIZv1(...) //printf(__VA_ARGS__) +#define DECOMPLOG_CODEWORDS_LIZv1(...) //printf(__VA_ARGS__) + + + + +/*-************************************ +* Common Constants +**************************************/ +#define MINMATCH 4 +//#define USE_LZ4_ONLY +//#define LIZARD_USE_TEST + +#define LIZARD_DICT_SIZE (1<<24) +#define WILDCOPYLENGTH 16 +#define LASTLITERALS WILDCOPYLENGTH +#define MFLIMIT (WILDCOPYLENGTH+MINMATCH) + +#define LIZARD_MAX_PRICE (1<<28) +#define LIZARD_INIT_LAST_OFFSET 0 +#define LIZARD_MAX_16BIT_OFFSET (1<<16) +#define MM_LONGOFF 16 +#define LIZARD_BLOCK_SIZE_PAD (LIZARD_BLOCK_SIZE+32) +#define LIZARD_COMPRESS_ADD_BUF (5*LIZARD_BLOCK_SIZE_PAD) +#ifndef LIZARD_NO_HUFFMAN + #define LIZARD_COMPRESS_ADD_HUF HUF_compressBound(LIZARD_BLOCK_SIZE_PAD) + #define LIZARD_HUF_BLOCK_SIZE LIZARD_BLOCK_SIZE +#else + #define LIZARD_COMPRESS_ADD_HUF 0 + #define LIZARD_HUF_BLOCK_SIZE 1 +#endif + +/* LZ4 codewords */ +#define ML_BITS_LZ4 4 +#define ML_MASK_LZ4 ((1U<= 1<<16 */ + U32 sufficientLength; /* used only by optimal parser: size of matches which is acceptable: larger == more compression, slower */ + U32 fullSearch; /* used only by optimal parser: perform full search of matches: 1 == more compression, slower */ + Lizard_parser_type parserType; + Lizard_decompress_type decompressType; +} Lizard_parameters; + + +struct Lizard_stream_s +{ + const BYTE* end; /* next block here to continue on current prefix */ + const BYTE* base; /* All index relative to this position */ + const BYTE* dictBase; /* alternate base for extDict */ + U32 dictLimit; /* below that point, need extDict */ + U32 lowLimit; /* below that point, no more dict */ + U32 nextToUpdate; /* index from which to continue dictionary update */ + U32 allocatedMemory; + int compressionLevel; + Lizard_parameters params; + U32 hashTableSize; + U32 chainTableSize; + U32* chainTable; + U32* hashTable; + int last_off; + const BYTE* off24pos; + U32 huffType; + U32 comprStreamLen; + + BYTE* huffBase; + BYTE* huffEnd; + BYTE* offset16Base; + BYTE* offset24Base; + BYTE* lenBase; + BYTE* literalsBase; + BYTE* flagsBase; + BYTE* offset16Ptr; + BYTE* offset24Ptr; + BYTE* lenPtr; + BYTE* literalsPtr; + BYTE* flagsPtr; + BYTE* offset16End; + BYTE* offset24End; + BYTE* lenEnd; + BYTE* literalsEnd; + BYTE* flagsEnd; + U32 flagFreq[256]; + U32 litFreq[256]; + U32 litSum, flagSum; + U32 litPriceSum, log2LitSum, log2FlagSum; + U32 cachedPrice; + U32 cachedLitLength; + const BYTE* cachedLiterals; + const BYTE* diffBase; + const BYTE* srcBase; + const BYTE* destBase; +}; + +struct Lizard_dstream_s +{ + const BYTE* offset16Ptr; + const BYTE* offset24Ptr; + const BYTE* lenPtr; + const BYTE* literalsPtr; + const BYTE* flagsPtr; + const BYTE* offset16End; + const BYTE* offset24End; + const BYTE* lenEnd; + const BYTE* literalsEnd; + const BYTE* flagsEnd; + const BYTE* diffBase; + intptr_t last_off; +}; + +typedef struct Lizard_dstream_s Lizard_dstream_t; + +/* ************************************* +* HC Pre-defined compression levels +***************************************/ +#define LIZARD_WINDOWLOG_LZ4 16 +#define LIZARD_CHAINLOG_LZ4 LIZARD_WINDOWLOG_LZ4 +#define LIZARD_HASHLOG_LZ4 18 +#define LIZARD_HASHLOG_LZ4SM 12 + +#define LIZARD_WINDOWLOG_LIZv1 22 +#define LIZARD_CHAINLOG_LIZv1 LIZARD_WINDOWLOG_LIZv1 +#define LIZARD_HASHLOG_LIZv1 18 + + + +static const Lizard_parameters Lizard_defaultParameters[LIZARD_MAX_CLEVEL+1-LIZARD_MIN_CLEVEL] = +{ + /* windLog, contentLog, HashLog, H3, Snum, SL, MMLongOff, SuffL, FS, Parser function, Decompressor type */ + { LIZARD_WINDOWLOG_LZ4, 0, LIZARD_HASHLOG_LZ4SM, 0, 0, 0, 0, 0, 0, Lizard_parser_fastSmall, Lizard_coderwords_LZ4 }, // level 10 + { LIZARD_WINDOWLOG_LZ4, 0, LIZARD_HASHLOG_LZ4, 0, 0, 0, 0, 0, 0, Lizard_parser_fast, Lizard_coderwords_LZ4 }, // level 11 + { LIZARD_WINDOWLOG_LZ4, 0, LIZARD_HASHLOG_LZ4, 0, 0, 0, 0, 0, 0, Lizard_parser_noChain, Lizard_coderwords_LZ4 }, // level 12 + { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 2, 5, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 13 + { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 4, 5, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 14 + { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 8, 5, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 15 + { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 16, 4, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 16 + { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 256, 4, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 17 + { LIZARD_WINDOWLOG_LZ4, LIZARD_WINDOWLOG_LZ4+1, LIZARD_HASHLOG_LZ4, 16, 16, 4, 0, 1<<10, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LZ4 }, // level 18 + { LIZARD_WINDOWLOG_LZ4, LIZARD_WINDOWLOG_LZ4+1, 23, 16, 256, 4, 0, 1<<10, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LZ4 }, // level 19 + /* windLog, contentLog, HashLog, H3, Snum, SL, MMLongOff, SuffL, FS, Parser function, Decompressor type */ + { LIZARD_WINDOWLOG_LIZv1, 0, 14, 0, 1, 5, MM_LONGOFF, 0, 0, Lizard_parser_fastBig, Lizard_coderwords_LIZv1 }, // level 20 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 14, 13, 1, 5, MM_LONGOFF, 0, 0, Lizard_parser_priceFast, Lizard_coderwords_LIZv1 }, // level 21 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, LIZARD_HASHLOG_LIZv1, 13, 1, 5, MM_LONGOFF, 0, 0, Lizard_parser_priceFast, Lizard_coderwords_LIZv1 }, // level 22 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, LIZARD_HASHLOG_LIZv1, 13, 1, 5, MM_LONGOFF, 64, 0, Lizard_parser_lowestPrice, Lizard_coderwords_LIZv1 }, // level 23 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 23, 16, 2, 5, MM_LONGOFF, 64, 0, Lizard_parser_lowestPrice, Lizard_coderwords_LIZv1 }, // level 24 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 23, 16, 8, 4, MM_LONGOFF, 64, 0, Lizard_parser_lowestPrice, Lizard_coderwords_LIZv1 }, // level 25 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1+1, 23, 16, 8, 4, MM_LONGOFF, 64, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 26 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1+1, 23, 16, 128, 4, MM_LONGOFF, 64, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 27 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1+1, 23, 24, 1<<10, 4, MM_LONGOFF, 1<<10, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 28 + { 24, 25, 23, 24, 1<<10, 4, MM_LONGOFF, 1<<10, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 29 +#ifndef LIZARD_NO_HUFFMAN + /* windLog, contentLog, HashLog, H3, Snum, SL, MMLongOff, SuffL, FS, Parser function, Decompressor type */ + { LIZARD_WINDOWLOG_LZ4, 0, LIZARD_HASHLOG_LZ4SM, 0, 0, 0, 0, 0, 0, Lizard_parser_fastSmall, Lizard_coderwords_LZ4 }, // level 30 + { LIZARD_WINDOWLOG_LZ4, 0, LIZARD_HASHLOG_LZ4, 0, 0, 0, 0, 0, 0, Lizard_parser_fast, Lizard_coderwords_LZ4 }, // level 31 + { LIZARD_WINDOWLOG_LZ4, 0, 14, 0, 0, 0, 0, 0, 0, Lizard_parser_noChain, Lizard_coderwords_LZ4 }, // level 32 + { LIZARD_WINDOWLOG_LZ4, 0, LIZARD_HASHLOG_LZ4, 0, 0, 0, 0, 0, 0, Lizard_parser_noChain, Lizard_coderwords_LZ4 }, // level 33 + { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 2, 5, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 34 + { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 4, 5, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 35 + { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 8, 5, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 36 + { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 16, 4, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 37 + { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 256, 4, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 38 + { LIZARD_WINDOWLOG_LZ4, LIZARD_WINDOWLOG_LZ4+1, 23, 16, 256, 4, 0, 1<<10, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LZ4 }, // level 39 + /* windLog, contentLog, HashLog, H3, Snum, SL, MMLongOff, SuffL, FS, Parser function, Decompressor type */ + { LIZARD_WINDOWLOG_LIZv1, 0, 14, 0, 1, 5, MM_LONGOFF, 0, 0, Lizard_parser_fastBig, Lizard_coderwords_LIZv1 }, // level 40 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 14, 13, 1, 5, MM_LONGOFF, 0, 0, Lizard_parser_priceFast, Lizard_coderwords_LIZv1 }, // level 41 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, LIZARD_HASHLOG_LIZv1, 13, 1, 5, MM_LONGOFF, 0, 0, Lizard_parser_priceFast, Lizard_coderwords_LIZv1 }, // level 42 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, LIZARD_HASHLOG_LIZv1, 13, 1, 5, MM_LONGOFF, 64, 0, Lizard_parser_lowestPrice, Lizard_coderwords_LIZv1 }, // level 43 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 23, 16, 2, 5, MM_LONGOFF, 64, 0, Lizard_parser_lowestPrice, Lizard_coderwords_LIZv1 }, // level 44 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 23, 16, 8, 4, MM_LONGOFF, 64, 0, Lizard_parser_lowestPrice, Lizard_coderwords_LIZv1 }, // level 45 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 23, 16, 8, 4, MM_LONGOFF, 64, 0, Lizard_parser_optimalPrice, Lizard_coderwords_LIZv1 }, // level 46 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1+1, 23, 16, 8, 4, MM_LONGOFF, 64, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 47 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1+1, 23, 16, 128, 4, MM_LONGOFF, 64, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 48 + { 24, 25, 23, 24, 1<<10, 4, MM_LONGOFF, 1<<10, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 49 +#endif +// { 10, 10, 10, 0, 0, 4, 0, 0, 0, Lizard_fast }, // min values +// { 24, 24, 28, 24, 1<<24, 7, 0, 1<<24, 2, Lizard_optimal_price }, // max values +}; + + + +/*-************************************ +* Compiler Options +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# include +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ +#else +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ +# if defined(__GNUC__) || defined(__clang__) +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +# else +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +#endif /* _MSC_VER */ + +#define LIZARD_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +#if (LIZARD_GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__) +# define expect(expr,value) (__builtin_expect ((expr),(value)) ) +#else +# define expect(expr,value) (expr) +#endif + +#define likely(expr) expect((expr) != 0, 1) +#define unlikely(expr) expect((expr) != 0, 0) + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define ALLOCATOR(n,s) calloc(n,s) +#define FREEMEM free +#define MEM_INIT memset +#ifndef MAX + #define MAX(a,b) ((a)>(b))?(a):(b) +#endif +#ifndef MIN + #define MIN(a,b) ((a)<(b)?(a):(b)) +#endif + +#if MINMATCH == 3 + #define MEM_readMINMATCH(ptr) (U32)(MEM_read32(ptr)<<8) +#else + #define MEM_readMINMATCH(ptr) (U32)(MEM_read32(ptr)) +#endif + + + + +/*-************************************ +* Reading and writing into memory +**************************************/ +#define STEPSIZE sizeof(size_t) + + +MEM_STATIC void Lizard_copy8(void* dst, const void* src) +{ + memcpy(dst,src,8); +} + +/* customized variant of memcpy, which can overwrite up to 7 bytes beyond dstEnd */ +MEM_STATIC void Lizard_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) +{ + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* const e = (BYTE*)dstEnd; + +#if 0 + const size_t l2 = 8 - (((size_t)d) & (sizeof(void*)-1)); + Lizard_copy8(d,s); if (d>e-9) return; + d+=l2; s+=l2; +#endif /* join to align */ + + do { Lizard_copy8(d,s); d+=8; s+=8; } while (d= 3) /* GCC Intrinsic */ + return 31 - __builtin_clz(val); +# else /* Software version */ + static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; + U32 v = val; + int r; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27]; + return r; +# endif +} + + +/*-************************************ +* Common functions +**************************************/ +MEM_STATIC unsigned Lizard_NbCommonBytes (register size_t val) +{ + if (MEM_isLittleEndian()) { + if (MEM_64bits()) { +# if defined(_MSC_VER) && defined(_WIN64) && !defined(LIZARD_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanForward64( &r, (U64)val ); + return (int)(r>>3); +# elif (defined(__clang__) || (LIZARD_GCC_VERSION >= 304)) && !defined(LIZARD_FORCE_SW_BITCOUNT) + return (__builtin_ctzll((U64)val) >> 3); +# else + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; +# endif + } else /* 32 bits */ { +# if defined(_MSC_VER) && !defined(LIZARD_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanForward( &r, (U32)val ); + return (int)(r>>3); +# elif (defined(__clang__) || (LIZARD_GCC_VERSION >= 304)) && !defined(LIZARD_FORCE_SW_BITCOUNT) + return (__builtin_ctz((U32)val) >> 3); +# else + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +# endif + } + } else /* Big Endian CPU */ { + if (MEM_64bits()) { +# if defined(_MSC_VER) && defined(_WIN64) && !defined(LIZARD_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse64( &r, val ); + return (unsigned)(r>>3); +# elif (defined(__clang__) || (LIZARD_GCC_VERSION >= 304)) && !defined(LIZARD_FORCE_SW_BITCOUNT) + return (__builtin_clzll((U64)val) >> 3); +# else + unsigned r; + if (!(val>>32)) { r=4; } else { r=0; val>>=32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +# endif + } else /* 32 bits */ { +# if defined(_MSC_VER) && !defined(LIZARD_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse( &r, (unsigned long)val ); + return (unsigned)(r>>3); +# elif (defined(__clang__) || (LIZARD_GCC_VERSION >= 304)) && !defined(LIZARD_FORCE_SW_BITCOUNT) + return (__builtin_clz((U32)val) >> 3); +# else + unsigned r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; +# endif + } + } +} + +MEM_STATIC unsigned Lizard_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) +{ + const BYTE* const pStart = pIn; + + while (likely(pIn + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOTLizard_hash4Ptr + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Lizard source repository : https://github.com/inikep/lizard +*/ + + +/* ************************************* +* Includes +***************************************/ +#include "lizard_compress.h" +#include "lizard_common.h" +#include +#include // intptr_t +#ifndef USE_LZ4_ONLY + #ifdef LIZARD_USE_TEST + #include "test/lizard_common_test.h" + #include "test/lizard_compress_test.h" + #else + #include "lizard_compress_liz.h" + #endif +#endif +#include "lizard_compress_lz4.h" +#include "../zstd/huf.h" + + +/* ************************************* +* Local Macros +***************************************/ +#define DELTANEXT(p) chainTable[(p) & contentMask] +#define LIZARD_MINIMAL_HUFF_GAIN(comprSize) (comprSize + (comprSize/8) + 512) +#define LIZARD_MINIMAL_BLOCK_GAIN(comprSize) (comprSize + (comprSize/32) + 512) + + +/*-************************************ +* Local Utils +**************************************/ +int Lizard_versionNumber (void) { return LIZARD_VERSION_NUMBER; } +int Lizard_compressBound(int isize) { return LIZARD_COMPRESSBOUND(isize); } +int Lizard_sizeofState_MinLevel() { return Lizard_sizeofState(LIZARD_MIN_CLEVEL); } + + + +/* ************************************* +* Hash functions +***************************************/ +#define HASH_UPDATE_LIMIT 8 /* equal to MEM_read64 */ +static const U32 prime4bytes = 2654435761U; +static const U64 prime5bytes = 889523592379ULL; +static const U64 prime6bytes = 227718039650203ULL; +static const U64 prime7bytes = 58295818150454627ULL; + +#if MINMATCH == 3 +static const U32 prime3bytes = 506832829U; +static U32 Lizard_hash3(U32 u, U32 h) { return (u * prime3bytes) << (32-24) >> (32-h) ; } +static size_t Lizard_hash3Ptr(const void* ptr, U32 h) { return Lizard_hash3(MEM_read32(ptr), h); } +#endif + +static U32 Lizard_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; } +static size_t Lizard_hash4Ptr(const void* ptr, U32 h) { return Lizard_hash4(MEM_read32(ptr), h); } + +static size_t Lizard_hash5(U64 u, U32 h) { return (size_t)((u * prime5bytes) << (64-40) >> (64-h)) ; } +static size_t Lizard_hash5Ptr(const void* p, U32 h) { return Lizard_hash5(MEM_read64(p), h); } + +static size_t Lizard_hash6(U64 u, U32 h) { return (size_t)((u * prime6bytes) << (64-48) >> (64-h)) ; } +static size_t Lizard_hash6Ptr(const void* p, U32 h) { return Lizard_hash6(MEM_read64(p), h); } + +static size_t Lizard_hash7(U64 u, U32 h) { return (size_t)((u * prime7bytes) << (64-56) >> (64-h)) ; } +static size_t Lizard_hash7Ptr(const void* p, U32 h) { return Lizard_hash7(MEM_read64(p), h); } + +static size_t Lizard_hashPtr(const void* p, U32 hBits, U32 mls) +{ + switch(mls) + { + default: + case 4: return Lizard_hash4Ptr(p, hBits); + case 5: return Lizard_hash5Ptr(p, hBits); + case 6: return Lizard_hash6Ptr(p, hBits); + case 7: return Lizard_hash7Ptr(p, hBits); + } +} + + + + +/************************************** +* Internal functions +**************************************/ +/** Lizard_count_2segments() : +* can count match length with `ip` & `match` in 2 different segments. +* convention : on reaching mEnd, match count continue starting from iStart +*/ +static size_t Lizard_count_2segments(const BYTE* ip, const BYTE* match, const BYTE* iEnd, const BYTE* mEnd, const BYTE* iStart) +{ + const BYTE* const vEnd = MIN( ip + (mEnd - match), iEnd); + size_t const matchLength = Lizard_count(ip, match, vEnd); + if (match + matchLength != mEnd) return matchLength; + return matchLength + Lizard_count(ip+matchLength, iStart, iEnd); +} + + +void Lizard_initBlock(Lizard_stream_t* ctx) +{ + ctx->offset16Ptr = ctx->offset16Base; + ctx->offset24Ptr = ctx->offset24Base; + ctx->lenPtr = ctx->lenBase; + ctx->literalsPtr = ctx->literalsBase; + ctx->flagsPtr = ctx->flagsBase; + ctx->last_off = LIZARD_INIT_LAST_OFFSET; /* reset last offset */ +} + + +FORCE_INLINE int Lizard_writeStream(int useHuff, Lizard_stream_t* ctx, BYTE* streamPtr, uint32_t streamLen, BYTE** op, BYTE* oend) +{ + if (useHuff && streamLen > 1024) { +#ifndef LIZARD_NO_HUFFMAN + int useHuffBuf; + if (*op + 6 > oend) { LIZARD_LOG_COMPRESS("*op[%p] + 6 > oend[%p]\n", *op, oend); return -1; } + + useHuffBuf = ((size_t)(oend - (*op + 6)) < HUF_compressBound(streamLen)); + if (useHuffBuf) { + if (streamLen > LIZARD_BLOCK_SIZE) { LIZARD_LOG_COMPRESS("streamLen[%d] > LIZARD_BLOCK_SIZE\n", streamLen); return -1; } + ctx->comprStreamLen = (U32)HUF_compress(ctx->huffBase, ctx->huffEnd - ctx->huffBase, streamPtr, streamLen); + } else { + ctx->comprStreamLen = (U32)HUF_compress(*op + 6, oend - (*op + 6), streamPtr, streamLen); + } + + if (!HUF_isError(ctx->comprStreamLen)) { + if (ctx->comprStreamLen > 0 && (LIZARD_MINIMAL_HUFF_GAIN(ctx->comprStreamLen) < streamLen)) { /* compressible */ + MEM_writeLE24(*op, streamLen); + MEM_writeLE24(*op+3, ctx->comprStreamLen); + if (useHuffBuf) { + if ((size_t)(oend - (*op + 6)) < ctx->comprStreamLen) { LIZARD_LOG_COMPRESS("*op[%p] oend[%p] comprStreamLen[%d]\n", *op, oend, (int)ctx->comprStreamLen); return -1; } + memcpy(*op + 6, ctx->huffBase, ctx->comprStreamLen); + } + *op += ctx->comprStreamLen + 6; + LIZARD_LOG_COMPRESS("HUF_compress streamLen=%d comprStreamLen=%d\n", (int)streamLen, (int)ctx->comprStreamLen); + return 1; + } else { LIZARD_LOG_COMPRESS("HUF_compress ERROR comprStreamLen=%d streamLen=%d\n", (int)ctx->comprStreamLen, (int)streamLen); } + } else { LIZARD_LOG_COMPRESS("HUF_compress ERROR %d: %s\n", (int)ctx->comprStreamLen, HUF_getErrorName(ctx->comprStreamLen)); } +#else + LIZARD_LOG_COMPRESS("compiled with LIZARD_NO_HUFFMAN\n"); + (void)ctx; + return -1; +#endif + } else ctx->comprStreamLen = 0; + + if (*op + 3 + streamLen > oend) { LIZARD_LOG_COMPRESS("*op[%p] + 3 + streamLen[%d] > oend[%p]\n", *op, streamLen, oend); return -1; } + MEM_writeLE24(*op, streamLen); + *op += 3; + memcpy(*op, streamPtr, streamLen); + *op += streamLen; + LIZARD_LOG_COMPRESS("Uncompressed streamLen=%d\n", (int)streamLen); + return 0; +} + + +int Lizard_writeBlock(Lizard_stream_t* ctx, const BYTE* ip, uint32_t inputSize, BYTE** op, BYTE* oend) +{ + int res; + uint32_t flagsLen = (uint32_t)(ctx->flagsPtr - ctx->flagsBase); + uint32_t literalsLen = (uint32_t)(ctx->literalsPtr - ctx->literalsBase); + uint32_t lenLen = (uint32_t)(ctx->lenPtr - ctx->lenBase); + uint32_t offset16Len = (uint32_t)(ctx->offset16Ptr - ctx->offset16Base); + uint32_t offset24Len = (uint32_t)(ctx->offset24Ptr - ctx->offset24Base); + uint32_t sum = flagsLen + literalsLen + lenLen + offset16Len + offset24Len; +#ifdef LIZARD_USE_LOGS + uint32_t comprFlagsLen, comprLiteralsLen; +#endif + + BYTE* start = *op; + + if ((literalsLen < WILDCOPYLENGTH) || (sum+5*3+1 > inputSize)) goto _write_uncompressed; + + *start = 0; + *op += 1; + + res = Lizard_writeStream(0, ctx, ctx->lenBase, lenLen, op, oend); + if (res < 0) goto _output_error; else *start += (BYTE)(res*LIZARD_FLAG_LEN); + + res = Lizard_writeStream(ctx->huffType&LIZARD_FLAG_OFFSET16, ctx, ctx->offset16Base, offset16Len, op, oend); + if (res < 0) goto _output_error; else *start += (BYTE)(res*LIZARD_FLAG_OFFSET16); + + res = Lizard_writeStream(ctx->huffType&LIZARD_FLAG_OFFSET24, ctx, ctx->offset24Base, offset24Len, op, oend); + if (res < 0) goto _output_error; else *start += (BYTE)(res*LIZARD_FLAG_OFFSET24); + + res = Lizard_writeStream(ctx->huffType&LIZARD_FLAG_FLAGS, ctx, ctx->flagsBase, flagsLen, op, oend); + if (res < 0) goto _output_error; else *start += (BYTE)(res*LIZARD_FLAG_FLAGS); +#ifdef LIZARD_USE_LOGS + comprFlagsLen = ctx->comprStreamLen; +#endif + + res = Lizard_writeStream(ctx->huffType&LIZARD_FLAG_LITERALS, ctx, ctx->literalsBase, literalsLen, op, oend); + if (res < 0) goto _output_error; else *start += (BYTE)(res*LIZARD_FLAG_LITERALS); +#ifdef LIZARD_USE_LOGS + comprLiteralsLen = ctx->comprStreamLen; + sum = (int)(*op-start); +#endif + + if (LIZARD_MINIMAL_BLOCK_GAIN((uint32_t)(*op-start)) > inputSize) goto _write_uncompressed; + + LIZARD_LOG_COMPRESS("%d: total=%d block=%d flagsLen[%.2f%%]=%d comprFlagsLen[%.2f%%]=%d literalsLen[%.2f%%]=%d comprLiteralsLen[%.2f%%]=%d lenLen=%d offset16Len[%.2f%%]=%d offset24Len[%.2f%%]=%d\n", (int)(ip - ctx->srcBase), + (int)(*op - ctx->destBase), sum, (flagsLen*100.0)/sum, flagsLen, (comprFlagsLen*100.0)/sum, comprFlagsLen, (literalsLen*100.0)/sum, literalsLen, (comprLiteralsLen*100.0)/sum, comprLiteralsLen, + lenLen, (offset16Len*100.0)/sum, offset16Len, (offset24Len*100.0)/sum, offset24Len); + return 0; + +_write_uncompressed: + LIZARD_LOG_COMPRESS("%d: total=%d block=%d UNCOMPRESSED inputSize=%u outSize=%d\n", (int)(ip - ctx->srcBase), + (int)(*op - ctx->destBase), (int)(*op-start), inputSize, (int)(oend-start)); + if ((uint32_t)(oend - start) < inputSize + 4) goto _output_error; + *start = LIZARD_FLAG_UNCOMPRESSED; + *op = start + 1; + MEM_writeLE24(*op, inputSize); + *op += 3; + memcpy(*op, ip, inputSize); + *op += inputSize; + return 0; + +_output_error: + LIZARD_LOG_COMPRESS("Lizard_writeBlock ERROR size=%d/%d flagsLen=%d literalsLen=%d lenLen=%d offset16Len=%d offset24Len=%d\n", (int)(*op-start), (int)(oend-start), flagsLen, literalsLen, lenLen, offset16Len, offset24Len); + return 1; +} + + +FORCE_INLINE int Lizard_encodeSequence ( + Lizard_stream_t* ctx, + const BYTE** ip, + const BYTE** anchor, + size_t matchLength, + const BYTE* const match) +{ +#ifdef USE_LZ4_ONLY + return Lizard_encodeSequence_LZ4(ctx, ip, anchor, matchLength, match); +#else + if (ctx->params.decompressType == Lizard_coderwords_LZ4) + return Lizard_encodeSequence_LZ4(ctx, ip, anchor, matchLength, match); + + return Lizard_encodeSequence_LIZv1(ctx, ip, anchor, matchLength, match); +#endif +} + + +FORCE_INLINE int Lizard_encodeLastLiterals ( + Lizard_stream_t* ctx, + const BYTE** ip, + const BYTE** anchor) +{ + LIZARD_LOG_COMPRESS("Lizard_encodeLastLiterals Lizard_coderwords_LZ4=%d\n", ctx->params.decompressType == Lizard_coderwords_LZ4); +#ifdef USE_LZ4_ONLY + return Lizard_encodeLastLiterals_LZ4(ctx, ip, anchor); +#else + if (ctx->params.decompressType == Lizard_coderwords_LZ4) + return Lizard_encodeLastLiterals_LZ4(ctx, ip, anchor); + + return Lizard_encodeLastLiterals_LIZv1(ctx, ip, anchor); +#endif +} + + +/************************************** +* Include parsers +**************************************/ +#include "lizard_parser_hashchain.h" +#include "lizard_parser_nochain.h" +#include "lizard_parser_fast.h" +#include "lizard_parser_fastsmall.h" +#include "lizard_parser_fastbig.h" +#ifndef USE_LZ4_ONLY + #include "lizard_parser_optimal.h" + #include "lizard_parser_lowestprice.h" + #include "lizard_parser_pricefast.h" +#endif + + +int Lizard_verifyCompressionLevel(int compressionLevel) +{ + if (compressionLevel > LIZARD_MAX_CLEVEL) compressionLevel = LIZARD_MAX_CLEVEL; + if (compressionLevel < LIZARD_MIN_CLEVEL) compressionLevel = LIZARD_DEFAULT_CLEVEL; + return compressionLevel; +} + + +int Lizard_sizeofState(int compressionLevel) +{ + Lizard_parameters params; + U32 hashTableSize, chainTableSize; + + compressionLevel = Lizard_verifyCompressionLevel(compressionLevel); + params = Lizard_defaultParameters[compressionLevel - LIZARD_MIN_CLEVEL]; +// hashTableSize = (U32)(sizeof(U32)*(((size_t)1 << params.hashLog3)+((size_t)1 << params.hashLog))); + hashTableSize = (U32)(sizeof(U32)*(((size_t)1 << params.hashLog))); + chainTableSize = (U32)(sizeof(U32)*((size_t)1 << params.contentLog)); + + return sizeof(Lizard_stream_t) + hashTableSize + chainTableSize + LIZARD_COMPRESS_ADD_BUF + (int)LIZARD_COMPRESS_ADD_HUF; +} + + +static void Lizard_init(Lizard_stream_t* ctx, const BYTE* start) +{ + // No need to use memset() on tables as values are always bound checked +#ifdef LIZARD_RESET_MEM + MEM_INIT((void*)ctx->hashTable, 0, ctx->hashTableSize); + MEM_INIT(ctx->chainTable, 0x01, ctx->chainTableSize); +#endif + // printf("memset hashTable=%p hashEnd=%p chainTable=%p chainEnd=%p\n", ctx->hashTable, ((BYTE*)ctx->hashTable) + ctx->hashTableSize, ctx->chainTable, ((BYTE*)ctx->chainTable)+ctx->chainTableSize); + ctx->nextToUpdate = LIZARD_DICT_SIZE; + ctx->base = start - LIZARD_DICT_SIZE; + ctx->end = start; + ctx->dictBase = start - LIZARD_DICT_SIZE; + ctx->dictLimit = LIZARD_DICT_SIZE; + ctx->lowLimit = LIZARD_DICT_SIZE; + ctx->last_off = LIZARD_INIT_LAST_OFFSET; + ctx->litSum = 0; +} + + +/* if ctx==NULL memory is allocated and returned as value */ +Lizard_stream_t* Lizard_initStream(Lizard_stream_t* ctx, int compressionLevel) +{ + Lizard_parameters params; + U32 hashTableSize, chainTableSize; + void *tempPtr; + + compressionLevel = Lizard_verifyCompressionLevel(compressionLevel); + params = Lizard_defaultParameters[compressionLevel - LIZARD_MIN_CLEVEL]; +// hashTableSize = (U32)(sizeof(U32)*(((size_t)1 << params.hashLog3)+((size_t)1 << params.hashLog))); + hashTableSize = (U32)(sizeof(U32)*(((size_t)1 << params.hashLog))); + chainTableSize = (U32)(sizeof(U32)*((size_t)1 << params.contentLog)); + + if (!ctx) + { + ctx = (Lizard_stream_t*)malloc(sizeof(Lizard_stream_t) + hashTableSize + chainTableSize + LIZARD_COMPRESS_ADD_BUF + LIZARD_COMPRESS_ADD_HUF); + if (!ctx) { printf("ERROR: Cannot allocate %d MB (compressionLevel=%d)\n", (int)(sizeof(Lizard_stream_t) + hashTableSize + chainTableSize)>>20, compressionLevel); return 0; } + LIZARD_LOG_COMPRESS("Allocated %d MB (compressionLevel=%d)\n", (int)(sizeof(Lizard_stream_t) + hashTableSize + chainTableSize)>>20, compressionLevel); + ctx->allocatedMemory = sizeof(Lizard_stream_t) + hashTableSize + chainTableSize + LIZARD_COMPRESS_ADD_BUF + (U32)LIZARD_COMPRESS_ADD_HUF; + // printf("malloc from=%p to=%p hashTable=%p hashEnd=%p chainTable=%p chainEnd=%p\n", ctx, ((BYTE*)ctx)+sizeof(Lizard_stream_t) + hashTableSize + chainTableSize, ctx->hashTable, ((BYTE*)ctx->hashTable) + hashTableSize, ctx->chainTable, ((BYTE*)ctx->chainTable)+chainTableSize); + } + + tempPtr = ctx; + ctx->hashTable = (U32*)(tempPtr) + sizeof(Lizard_stream_t)/4; + ctx->hashTableSize = hashTableSize; + ctx->chainTable = ctx->hashTable + hashTableSize/4; + ctx->chainTableSize = chainTableSize; + ctx->params = params; + ctx->compressionLevel = (unsigned)compressionLevel; + if (compressionLevel < 30) + ctx->huffType = 0; + else + ctx->huffType = LIZARD_FLAG_LITERALS + LIZARD_FLAG_FLAGS; // + LIZARD_FLAG_OFFSET16 + LIZARD_FLAG_OFFSET24; + + ctx->literalsBase = (BYTE*)ctx->hashTable + ctx->hashTableSize + ctx->chainTableSize; + ctx->flagsBase = ctx->literalsEnd = ctx->literalsBase + LIZARD_BLOCK_SIZE_PAD; + ctx->lenBase = ctx->flagsEnd = ctx->flagsBase + LIZARD_BLOCK_SIZE_PAD; + ctx->offset16Base = ctx->lenEnd = ctx->lenBase + LIZARD_BLOCK_SIZE_PAD; + ctx->offset24Base = ctx->offset16End = ctx->offset16Base + LIZARD_BLOCK_SIZE_PAD; + ctx->huffBase = ctx->offset24End = ctx->offset24Base + LIZARD_BLOCK_SIZE_PAD; + ctx->huffEnd = ctx->huffBase + LIZARD_COMPRESS_ADD_HUF; + + return ctx; +} + + + +Lizard_stream_t* Lizard_createStream(int compressionLevel) +{ + Lizard_stream_t* ctx = Lizard_initStream(NULL, compressionLevel); + return ctx; +} + + +/* initialization */ +Lizard_stream_t* Lizard_resetStream(Lizard_stream_t* ctx, int compressionLevel) +{ + size_t wanted = Lizard_sizeofState(compressionLevel); + + if (ctx->allocatedMemory < wanted) { + Lizard_freeStream(ctx); + ctx = Lizard_createStream(compressionLevel); + } else { + Lizard_initStream(ctx, compressionLevel); + } + + if (ctx) ctx->base = NULL; + return ctx; +} + + +int Lizard_freeStream(Lizard_stream_t* ctx) +{ + if (ctx) { + free(ctx); + } + return 0; +} + + +int Lizard_loadDict(Lizard_stream_t* Lizard_streamPtr, const char* dictionary, int dictSize) +{ + Lizard_stream_t* ctxPtr = (Lizard_stream_t*) Lizard_streamPtr; + if (dictSize > LIZARD_DICT_SIZE) { + dictionary += dictSize - LIZARD_DICT_SIZE; + dictSize = LIZARD_DICT_SIZE; + } + Lizard_init (ctxPtr, (const BYTE*)dictionary); + if (dictSize >= HASH_UPDATE_LIMIT) Lizard_Insert (ctxPtr, (const BYTE*)dictionary + (dictSize - (HASH_UPDATE_LIMIT-1))); + ctxPtr->end = (const BYTE*)dictionary + dictSize; + return dictSize; +} + + +static void Lizard_setExternalDict(Lizard_stream_t* ctxPtr, const BYTE* newBlock) +{ + if (ctxPtr->end >= ctxPtr->base + HASH_UPDATE_LIMIT) Lizard_Insert (ctxPtr, ctxPtr->end - (HASH_UPDATE_LIMIT-1)); /* Referencing remaining dictionary content */ + /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ + ctxPtr->lowLimit = ctxPtr->dictLimit; + ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); + ctxPtr->dictBase = ctxPtr->base; + ctxPtr->base = newBlock - ctxPtr->dictLimit; + ctxPtr->end = newBlock; + ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ +} + + +/* dictionary saving */ +int Lizard_saveDict (Lizard_stream_t* Lizard_streamPtr, char* safeBuffer, int dictSize) +{ + Lizard_stream_t* const ctx = (Lizard_stream_t*)Lizard_streamPtr; + int const prefixSize = (int)(ctx->end - (ctx->base + ctx->dictLimit)); + if (dictSize > LIZARD_DICT_SIZE) dictSize = LIZARD_DICT_SIZE; + if (dictSize < 4) dictSize = 0; + if (dictSize > prefixSize) dictSize = prefixSize; + memmove(safeBuffer, ctx->end - dictSize, dictSize); + { U32 const endIndex = (U32)(ctx->end - ctx->base); + ctx->end = (const BYTE*)safeBuffer + dictSize; + ctx->base = ctx->end - endIndex; + ctx->dictLimit = endIndex - dictSize; + ctx->lowLimit = endIndex - dictSize; + if (ctx->nextToUpdate < ctx->dictLimit) ctx->nextToUpdate = ctx->dictLimit; + } + return dictSize; +} + +FORCE_INLINE int Lizard_compress_generic ( + void* ctxvoid, + const char* source, + char* dest, + int inputSize, + int maxOutputSize) +{ + Lizard_stream_t* ctx = (Lizard_stream_t*) ctxvoid; + size_t dictSize = (size_t)(ctx->end - ctx->base) - ctx->dictLimit; + const BYTE* ip = (const BYTE*) source; + BYTE* op = (BYTE*) dest; + BYTE* const oend = op + maxOutputSize; + int res; + + (void)dictSize; + LIZARD_LOG_COMPRESS("Lizard_compress_generic source=%p inputSize=%d dest=%p maxOutputSize=%d cLevel=%d dictBase=%p dictSize=%d\n", source, inputSize, dest, maxOutputSize, ctx->compressionLevel, ctx->dictBase, (int)dictSize); + *op++ = (BYTE)ctx->compressionLevel; + maxOutputSize--; // can be lower than 0 + ctx->end += inputSize; + ctx->srcBase = ctx->off24pos = ip; + ctx->destBase = (BYTE*)dest; + + while (inputSize > 0) + { + int inputPart = MIN(LIZARD_BLOCK_SIZE, inputSize); + + if (ctx->huffType) Lizard_rescaleFreqs(ctx); + Lizard_initBlock(ctx); + ctx->diffBase = ip; + + switch(ctx->params.parserType) + { + default: + case Lizard_parser_fastSmall: + res = Lizard_compress_fastSmall(ctx, ip, ip+inputPart); break; + case Lizard_parser_fast: + res = Lizard_compress_fast(ctx, ip, ip+inputPart); break; + case Lizard_parser_noChain: + res = Lizard_compress_noChain(ctx, ip, ip+inputPart); break; + case Lizard_parser_hashChain: + res = Lizard_compress_hashChain(ctx, ip, ip+inputPart); break; +#ifndef USE_LZ4_ONLY + case Lizard_parser_fastBig: + res = Lizard_compress_fastBig(ctx, ip, ip+inputPart); break; + case Lizard_parser_priceFast: + res = Lizard_compress_priceFast(ctx, ip, ip+inputPart); break; + case Lizard_parser_lowestPrice: + res = Lizard_compress_lowestPrice(ctx, ip, ip+inputPart); break; + case Lizard_parser_optimalPrice: + case Lizard_parser_optimalPriceBT: + res = Lizard_compress_optimalPrice(ctx, ip, ip+inputPart); break; +#else + case Lizard_parser_priceFast: + case Lizard_parser_lowestPrice: + case Lizard_parser_optimalPrice: + case Lizard_parser_optimalPriceBT: + res = 0; +#endif + } + + LIZARD_LOG_COMPRESS("Lizard_compress_generic res=%d inputPart=%d \n", res, inputPart); + if (res <= 0) return res; + + if (Lizard_writeBlock(ctx, ip, inputPart, &op, oend)) goto _output_error; + + ip += inputPart; + inputSize -= inputPart; + LIZARD_LOG_COMPRESS("Lizard_compress_generic in=%d out=%d\n", (int)(ip-(const BYTE*)source), (int)(op-(BYTE*)dest)); + } + + LIZARD_LOG_COMPRESS("Lizard_compress_generic total=%d\n", (int)(op-(BYTE*)dest)); + return (int)(op-(BYTE*)dest); +_output_error: + LIZARD_LOG_COMPRESS("Lizard_compress_generic ERROR\n"); + return 0; +} + + +int Lizard_compress_continue (Lizard_stream_t* ctxPtr, + const char* source, char* dest, + int inputSize, int maxOutputSize) +{ + /* auto-init if forgotten */ + if (ctxPtr->base == NULL) Lizard_init (ctxPtr, (const BYTE*) source); + + /* Check overflow */ + if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) { + size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit; + if (dictSize > LIZARD_DICT_SIZE) dictSize = LIZARD_DICT_SIZE; + Lizard_loadDict((Lizard_stream_t*)ctxPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize); + } + + /* Check if blocks follow each other */ + if ((const BYTE*)source != ctxPtr->end) + Lizard_setExternalDict(ctxPtr, (const BYTE*)source); + + /* Check overlapping input/dictionary space */ + { const BYTE* sourceEnd = (const BYTE*) source + inputSize; + const BYTE* const dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit; + const BYTE* const dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit; + if ((sourceEnd > dictBegin) && ((const BYTE*)source < dictEnd)) { + if (sourceEnd > dictEnd) sourceEnd = dictEnd; + ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase); + if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit; + } + } + + return Lizard_compress_generic (ctxPtr, source, dest, inputSize, maxOutputSize); +} + + +int Lizard_compress_extState (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel) +{ + Lizard_stream_t* ctx = (Lizard_stream_t*) state; + if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ + + /* initialize stream */ + Lizard_initStream(ctx, compressionLevel); + Lizard_init ((Lizard_stream_t*)state, (const BYTE*)src); + + return Lizard_compress_generic (state, src, dst, srcSize, maxDstSize); +} + + +int Lizard_compress(const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel) +{ + int cSize; + Lizard_stream_t* statePtr = Lizard_createStream(compressionLevel); + + if (!statePtr) return 0; + cSize = Lizard_compress_extState(statePtr, src, dst, srcSize, maxDstSize, compressionLevel); + + Lizard_freeStream(statePtr); + return cSize; +} + + +/************************************** +* Level1 functions +**************************************/ +int Lizard_compress_extState_MinLevel(void* state, const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return Lizard_compress_extState(state, source, dest, inputSize, maxOutputSize, LIZARD_MIN_CLEVEL); +} + +int Lizard_compress_MinLevel(const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return Lizard_compress(source, dest, inputSize, maxOutputSize, LIZARD_MIN_CLEVEL); +} + +Lizard_stream_t* Lizard_createStream_MinLevel(void) +{ + return Lizard_createStream(LIZARD_MIN_CLEVEL); +} + +Lizard_stream_t* Lizard_resetStream_MinLevel(Lizard_stream_t* Lizard_stream) +{ + return Lizard_resetStream (Lizard_stream, LIZARD_MIN_CLEVEL); +} diff --git a/C/lizard/lizard_compress.h b/C/lizard/lizard_compress.h new file mode 100644 index 00000000..d84246e2 --- /dev/null +++ b/C/lizard/lizard_compress.h @@ -0,0 +1,208 @@ +/* + Lizard - Fast LZ compression algorithm + Header File + Copyright (C) 2011-2016, Yann Collet + Copyright (C) 2016-2017, Przemyslaw Skibinski + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Lizard source repository : https://github.com/inikep/lizard +*/ +#ifndef LIZARD_H_2983 +#define LIZARD_H_2983 + +#if defined (__cplusplus) +extern "C" { +#endif + +/* + * lizard_compress.h provides block compression functions. It gives full buffer control to user. + * Block compression functions are not-enough to send information, + * since it's still necessary to provide metadata (such as compressed size), + * and each application can do it in whichever way it wants. + * For interoperability, there is Lizard frame specification (lizard_Frame_format.md). + * A library is provided to take care of it, see lizard_frame.h. +*/ + + +/*^*************************************************************** +* Export parameters +*****************************************************************/ +/* +* LIZARD_DLL_EXPORT : +* Enable exporting of functions when building a Windows DLL +*/ +#if defined(LIZARD_DLL_EXPORT) && (LIZARD_DLL_EXPORT==1) +# define LIZARDLIB_API __declspec(dllexport) +#elif defined(LIZARD_DLL_IMPORT) && (LIZARD_DLL_IMPORT==1) +# define LIZARDLIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define LIZARDLIB_API +#endif + + +/*-************************************ +* Version +**************************************/ +#define LIZARD_VERSION_MAJOR 1 /* for breaking interface changes */ +#define LIZARD_VERSION_MINOR 0 /* for new (non-breaking) interface capabilities */ +#define LIZARD_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ + +#define LIZARD_VERSION_NUMBER (LIZARD_VERSION_MAJOR *100*100 + LIZARD_VERSION_MINOR *100 + LIZARD_VERSION_RELEASE) +int Lizard_versionNumber (void); + +#define LIZARD_LIB_VERSION LIZARD_VERSION_MAJOR.LIZARD_VERSION_MINOR.LIZARD_VERSION_RELEASE +#define LIZARD_QUOTE(str) #str +#define LIZARD_EXPAND_AND_QUOTE(str) LIZARD_QUOTE(str) +#define LIZARD_VERSION_STRING LIZARD_EXPAND_AND_QUOTE(LIZARD_LIB_VERSION) +const char* Lizard_versionString (void); + +typedef struct Lizard_stream_s Lizard_stream_t; + +#define LIZARD_MIN_CLEVEL 10 /* minimum compression level */ +#ifndef LIZARD_NO_HUFFMAN + #define LIZARD_MAX_CLEVEL 49 /* maximum compression level */ +#else + #define LIZARD_MAX_CLEVEL 29 /* maximum compression level */ +#endif +#define LIZARD_DEFAULT_CLEVEL 17 + + +/*-************************************ +* Simple Functions +**************************************/ + +LIZARDLIB_API int Lizard_compress (const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); + +/* +Lizard_compress() : + Compresses 'sourceSize' bytes from buffer 'source' + into already allocated 'dest' buffer of size 'maxDestSize'. + Compression is guaranteed to succeed if 'maxDestSize' >= Lizard_compressBound(sourceSize). + It also runs faster, so it's a recommended setting. + If the function cannot compress 'source' into a more limited 'dest' budget, + compression stops *immediately*, and the function result is zero. + As a consequence, 'dest' content is not valid. + This function never writes outside 'dest' buffer, nor read outside 'source' buffer. + sourceSize : Max supported value is LIZARD_MAX_INPUT_VALUE + maxDestSize : full or partial size of buffer 'dest' (which must be already allocated) + return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize) + or 0 if compression fails +*/ + + +/*-************************************ +* Advanced Functions +**************************************/ +#define LIZARD_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ +#define LIZARD_BLOCK_SIZE (1<<17) +#define LIZARD_BLOCK64K_SIZE (1<<16) +#define LIZARD_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LIZARD_MAX_INPUT_SIZE ? 0 : (isize) + 1 + 1 + ((isize/LIZARD_BLOCK_SIZE)+1)*4) + + +/*! +Lizard_compressBound() : + Provides the maximum size that Lizard compression may output in a "worst case" scenario (input data not compressible) + This function is primarily useful for memory allocation purposes (destination buffer size). + Macro LIZARD_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). + Note that Lizard_compress() compress faster when dest buffer size is >= Lizard_compressBound(srcSize) + inputSize : max supported value is LIZARD_MAX_INPUT_SIZE + return : maximum output size in a "worst case" scenario + or 0, if input size is too large ( > LIZARD_MAX_INPUT_SIZE) +*/ +LIZARDLIB_API int Lizard_compressBound(int inputSize); + + +/*! +Lizard_compress_extState() : + Same compression function, just using an externally allocated memory space to store compression state. + Use Lizard_sizeofState() to know how much memory must be allocated, + and allocate it on 8-bytes boundaries (using malloc() typically). + Then, provide it as 'void* state' to compression function. +*/ +LIZARDLIB_API int Lizard_sizeofState(int compressionLevel); + +LIZARDLIB_API int Lizard_compress_extState(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); + + + +/*-********************************************* +* Streaming Compression Functions +***********************************************/ + +/*! Lizard_createStream() will allocate and initialize an `Lizard_stream_t` structure. + * Lizard_freeStream() releases its memory. + * In the context of a DLL (liblizard), please use these methods rather than the static struct. + * They are more future proof, in case of a change of `Lizard_stream_t` size. + */ +LIZARDLIB_API Lizard_stream_t* Lizard_createStream(int compressionLevel); +LIZARDLIB_API int Lizard_freeStream (Lizard_stream_t* streamPtr); + + +/*! Lizard_resetStream() : + * Use this function to reset/reuse an allocated `Lizard_stream_t` structure + */ +LIZARDLIB_API Lizard_stream_t* Lizard_resetStream (Lizard_stream_t* streamPtr, int compressionLevel); + + +/*! Lizard_loadDict() : + * Use this function to load a static dictionary into Lizard_stream. + * Any previous data will be forgotten, only 'dictionary' will remain in memory. + * Loading a size of 0 is allowed. + * Return : dictionary size, in bytes (necessarily <= LIZARD_DICT_SIZE) + */ +LIZARDLIB_API int Lizard_loadDict (Lizard_stream_t* streamPtr, const char* dictionary, int dictSize); + + +/*! Lizard_compress_continue() : + * Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio. + * Important : Previous data blocks are assumed to still be present and unmodified ! + * 'dst' buffer must be already allocated. + * If maxDstSize >= Lizard_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. + * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero. + */ +LIZARDLIB_API int Lizard_compress_continue (Lizard_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize); + + +/*! Lizard_saveDict() : + * If previously compressed data block is not guaranteed to remain available at its memory location, + * save it into a safer place (char* safeBuffer). + * Note : you don't need to call Lizard_loadDict() afterwards, + * dictionary is immediately usable, you can therefore call Lizard_compress_continue(). + * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. + */ +LIZARDLIB_API int Lizard_saveDict (Lizard_stream_t* streamPtr, char* safeBuffer, int dictSize); + + + + + +#if defined (__cplusplus) +} +#endif + +#endif /* LIZARD_H_2983827168210 */ diff --git a/C/lizard/liz_compress_lz5v2.h b/C/lizard/lizard_compress_liz.h similarity index 60% rename from C/lizard/liz_compress_lz5v2.h rename to C/lizard/lizard_compress_liz.h index 8f1f686a..f531f367 100644 --- a/C/lizard/liz_compress_lz5v2.h +++ b/C/lizard/lizard_compress_liz.h @@ -1,13 +1,13 @@ -#define LIZ_FREQ_DIV 5 +#define LIZARD_FREQ_DIV 5 -FORCE_INLINE void LIZ_setLog2Prices(LIZ_stream_t* ctx) +FORCE_INLINE void Lizard_setLog2Prices(Lizard_stream_t* ctx) { - ctx->log2LitSum = LIZ_highbit32(ctx->litSum+1); - ctx->log2FlagSum = LIZ_highbit32(ctx->flagSum+1); + ctx->log2LitSum = Lizard_highbit32(ctx->litSum+1); + ctx->log2FlagSum = Lizard_highbit32(ctx->flagSum+1); } -MEM_STATIC void LIZ_rescaleFreqs(LIZ_stream_t* ctx) +MEM_STATIC void Lizard_rescaleFreqs(Lizard_stream_t* ctx) { unsigned u; @@ -29,19 +29,19 @@ MEM_STATIC void LIZ_rescaleFreqs(LIZ_stream_t* ctx) ctx->flagSum = 0; for (u=0; u < 256; u++) { - ctx->litFreq[u] = 1 + (ctx->litFreq[u]>>LIZ_FREQ_DIV); + ctx->litFreq[u] = 1 + (ctx->litFreq[u]>>LIZARD_FREQ_DIV); ctx->litSum += ctx->litFreq[u]; - ctx->flagFreq[u] = 1 + (ctx->flagFreq[u]>>LIZ_FREQ_DIV); + ctx->flagFreq[u] = 1 + (ctx->flagFreq[u]>>LIZARD_FREQ_DIV); ctx->flagSum += ctx->flagFreq[u]; } } - LIZ_setLog2Prices(ctx); + Lizard_setLog2Prices(ctx); } -FORCE_INLINE int LIZ_encodeSequence_LZ5v2 ( - LIZ_stream_t* ctx, +FORCE_INLINE int Lizard_encodeSequence_LIZv1 ( + Lizard_stream_t* ctx, const BYTE** ip, const BYTE** anchor, size_t matchLength, @@ -51,9 +51,9 @@ FORCE_INLINE int LIZ_encodeSequence_LZ5v2 ( size_t length = (size_t)(*ip - *anchor); BYTE* token = (ctx->flagsPtr)++; - if (length > 0 || offset < LIZ_MAX_16BIT_OFFSET) { + if (length > 0 || offset < LIZARD_MAX_16BIT_OFFSET) { /* Encode Literal length */ - // if ((limitedOutputBuffer) && (ctx->literalsPtr > oend - length - LIZ_LENGTH_SIZE_LZ5v2(length) - WILDCOPYLENGTH)) { LIZ_LOG_COMPRESS_LZ5v2("encodeSequence overflow1\n"); return 1; } /* Check output limit */ + // if ((limitedOutputBuffer) && (ctx->literalsPtr > oend - length - LIZARD_LENGTH_SIZE_LIZv1(length) - WILDCOPYLENGTH)) { LIZARD_LOG_COMPRESS_LIZv1("encodeSequence overflow1\n"); return 1; } /* Check output limit */ if (length >= MAX_SHORT_LITLEN) { size_t len; *token = MAX_SHORT_LITLEN; @@ -65,14 +65,14 @@ FORCE_INLINE int LIZ_encodeSequence_LZ5v2 ( else *token = (BYTE)length; /* Copy Literals */ - LIZ_wildCopy(ctx->literalsPtr, *anchor, (ctx->literalsPtr) + length); -#ifndef LIZ_NO_HUFFMAN + Lizard_wildCopy(ctx->literalsPtr, *anchor, (ctx->literalsPtr) + length); +#ifndef LIZARD_NO_HUFFMAN if (ctx->huffType) { ctx->litSum += (U32)length; ctx->litPriceSum += (U32)(length * ctx->log2LitSum); { U32 u; for (u=0; u < length; u++) { - ctx->litPriceSum -= LIZ_highbit32(ctx->litFreq[ctx->literalsPtr[u]]+1); + ctx->litPriceSum -= Lizard_highbit32(ctx->litFreq[ctx->literalsPtr[u]]+1); ctx->litFreq[ctx->literalsPtr[u]]++; } } } @@ -80,10 +80,10 @@ FORCE_INLINE int LIZ_encodeSequence_LZ5v2 ( ctx->literalsPtr += length; - if (offset >= LIZ_MAX_16BIT_OFFSET) { - COMPLOG_CODEWORDS_LZ5v2("T32+ literal=%u match=%u offset=%d\n", (U32)length, 0, 0); + if (offset >= LIZARD_MAX_16BIT_OFFSET) { + COMPLOG_CODEWORDS_LIZv1("T32+ literal=%u match=%u offset=%d\n", (U32)length, 0, 0); *token+=(1<huffType) { ctx->flagFreq[*token]++; ctx->flagSum++; @@ -94,23 +94,23 @@ FORCE_INLINE int LIZ_encodeSequence_LZ5v2 ( } /* Encode Offset */ - if (offset >= LIZ_MAX_16BIT_OFFSET) // 24-bit offset + if (offset >= LIZARD_MAX_16BIT_OFFSET) // 24-bit offset { - if (matchLength < MM_LONGOFF) printf("ERROR matchLength=%d/%d\n", (int)matchLength, MM_LONGOFF), exit(0); + if (matchLength < MM_LONGOFF) printf("ERROR matchLength=%d/%d\n", (int)matchLength, MM_LONGOFF), exit(1); - // if ((limitedOutputBuffer) && (ctx->literalsPtr > oend - 8 /*LIZ_LENGTH_SIZE_LZ5v2(length)*/)) { LIZ_LOG_COMPRESS_LZ5v2("encodeSequence overflow2\n"); return 1; } /* Check output limit */ - if (matchLength - MM_LONGOFF >= LIZ_LAST_LONG_OFF) + // if ((limitedOutputBuffer) && (ctx->literalsPtr > oend - 8 /*LIZARD_LENGTH_SIZE_LIZv1(length)*/)) { LIZARD_LOG_COMPRESS_LIZv1("encodeSequence overflow2\n"); return 1; } /* Check output limit */ + if (matchLength - MM_LONGOFF >= LIZARD_LAST_LONG_OFF) { - size_t len = matchLength - MM_LONGOFF - LIZ_LAST_LONG_OFF; - *token = LIZ_LAST_LONG_OFF; + size_t len = matchLength - MM_LONGOFF - LIZARD_LAST_LONG_OFF; + *token = LIZARD_LAST_LONG_OFF; if (len >= (1<<16)) { *(ctx->literalsPtr) = 255; MEM_writeLE24(ctx->literalsPtr+1, (U32)(len)); ctx->literalsPtr += 4; } else if (len >= 254) { *(ctx->literalsPtr) = 254; MEM_writeLE16(ctx->literalsPtr+1, (U16)(len)); ctx->literalsPtr += 3; } else *(ctx->literalsPtr)++ = (BYTE)len; - COMPLOG_CODEWORDS_LZ5v2("T31 literal=%u match=%u offset=%d\n", 0, (U32)matchLength, offset); + COMPLOG_CODEWORDS_LIZv1("T31 literal=%u match=%u offset=%d\n", 0, (U32)matchLength, offset); } else { - COMPLOG_CODEWORDS_LZ5v2("T0-30 literal=%u match=%u offset=%d\n", 0, (U32)matchLength, offset); + COMPLOG_CODEWORDS_LIZv1("T0-30 literal=%u match=%u offset=%d\n", 0, (U32)matchLength, offset); *token = (BYTE)(matchLength - MM_LONGOFF); } @@ -121,15 +121,16 @@ FORCE_INLINE int LIZ_encodeSequence_LZ5v2 ( } else { - COMPLOG_CODEWORDS_LZ5v2("T32+ literal=%u match=%u offset=%d\n", (U32)length, (U32)matchLength, offset); + COMPLOG_CODEWORDS_LIZv1("T32+ literal=%u match=%u offset=%d\n", (U32)length, (U32)matchLength, offset); if (offset == 0) { *token+=(1<last_off); exit(1); } + // it should never happen + if (offset < 8) { printf("ERROR offset=%d\n", (int)offset); exit(1); } + if (matchLength < MINMATCH) { printf("ERROR matchLength[%d] < MINMATCH offset=%d\n", (int)matchLength, (int)ctx->last_off); exit(1); } ctx->last_off = offset; MEM_writeLE16(ctx->offset16Ptr, (U16)ctx->last_off); ctx->offset16Ptr += 2; @@ -137,22 +138,22 @@ FORCE_INLINE int LIZ_encodeSequence_LZ5v2 ( /* Encode MatchLength */ length = matchLength; - // if ((limitedOutputBuffer) && (ctx->literalsPtr > oend - 5 /*LIZ_LENGTH_SIZE_LZ5v2(length)*/)) { LIZ_LOG_COMPRESS_LZ5v2("encodeSequence overflow2\n"); return 1; } /* Check output limit */ + // if ((limitedOutputBuffer) && (ctx->literalsPtr > oend - 5 /*LIZARD_LENGTH_SIZE_LIZv1(length)*/)) { LIZARD_LOG_COMPRESS_LIZv1("encodeSequence overflow2\n"); return 1; } /* Check output limit */ if (length >= MAX_SHORT_MATCHLEN) { - *token += (BYTE)(MAX_SHORT_MATCHLEN<= (1<<16)) { *(ctx->literalsPtr) = 255; MEM_writeLE24(ctx->literalsPtr+1, (U32)(length)); ctx->literalsPtr += 4; } else if (length >= 254) { *(ctx->literalsPtr) = 254; MEM_writeLE16(ctx->literalsPtr+1, (U16)(length)); ctx->literalsPtr += 3; } else *(ctx->literalsPtr)++ = (BYTE)length; } - else *token += (BYTE)(length<huffType) { ctx->flagFreq[*token]++; ctx->flagSum++; - LIZ_setLog2Prices(ctx); + Lizard_setLog2Prices(ctx); } #endif @@ -164,8 +165,8 @@ FORCE_INLINE int LIZ_encodeSequence_LZ5v2 ( } -FORCE_INLINE int LIZ_encodeLastLiterals_LZ5v2 ( - LIZ_stream_t* ctx, +FORCE_INLINE int Lizard_encodeLastLiterals_LIZv1 ( + Lizard_stream_t* ctx, const BYTE** ip, const BYTE** anchor) { @@ -178,31 +179,30 @@ FORCE_INLINE int LIZ_encodeLastLiterals_LZ5v2 ( } -#define LIZ_PRICE_MULT 1 -#define LIZ_GET_TOKEN_PRICE_LZ5v2(token) (LIZ_PRICE_MULT * (ctx->log2FlagSum - LIZ_highbit32(ctx->flagFreq[token]+1))) +#define LIZARD_PRICE_MULT 1 +#define LIZARD_GET_TOKEN_PRICE_LIZv1(token) (LIZARD_PRICE_MULT * (ctx->log2FlagSum - Lizard_highbit32(ctx->flagFreq[token]+1))) -FORCE_INLINE size_t LIZ_get_price_LZ5v2(LIZ_stream_t* const ctx, int rep, const BYTE *ip, const BYTE *off24pos, size_t litLength, U32 offset, size_t matchLength) +FORCE_INLINE size_t Lizard_get_price_LIZv1(Lizard_stream_t* const ctx, int rep, const BYTE *ip, const BYTE *off24pos, size_t litLength, U32 offset, size_t matchLength) { size_t price = 0; BYTE token = 0; -#ifndef LIZ_NO_HUFFMAN +#ifndef LIZARD_NO_HUFFMAN const BYTE* literals = ip - litLength; U32 u; - if ((ctx->huffType) && (ctx->params.parserType != LIZ_parser_lowestPrice)) { + if ((ctx->huffType) && (ctx->params.parserType != Lizard_parser_lowestPrice)) { if (ctx->cachedLiterals == literals && litLength >= ctx->cachedLitLength) { size_t const additional = litLength - ctx->cachedLitLength; - // printf("%d ", (int)litLength - (int)ctx->cachedLitLength); const BYTE* literals2 = ctx->cachedLiterals + ctx->cachedLitLength; - price = ctx->cachedPrice + LIZ_PRICE_MULT * additional * ctx->log2LitSum; + price = ctx->cachedPrice + LIZARD_PRICE_MULT * additional * ctx->log2LitSum; for (u=0; u < additional; u++) - price -= LIZ_PRICE_MULT * LIZ_highbit32(ctx->litFreq[literals2[u]]+1); + price -= LIZARD_PRICE_MULT * Lizard_highbit32(ctx->litFreq[literals2[u]]+1); ctx->cachedPrice = (U32)price; ctx->cachedLitLength = (U32)litLength; } else { - price = LIZ_PRICE_MULT * litLength * ctx->log2LitSum; + price = LIZARD_PRICE_MULT * litLength * ctx->log2LitSum; for (u=0; u < litLength; u++) - price -= LIZ_PRICE_MULT * LIZ_highbit32(ctx->litFreq[literals[u]]+1); + price -= LIZARD_PRICE_MULT * Lizard_highbit32(ctx->litFreq[literals[u]]+1); if (litLength >= 12) { ctx->cachedLiterals = literals; @@ -222,7 +222,7 @@ FORCE_INLINE size_t LIZ_get_price_LZ5v2(LIZ_stream_t* const ctx, int rep, const (void)off24pos; (void)rep; - if (litLength > 0 || offset < LIZ_MAX_16BIT_OFFSET) { + if (litLength > 0 || offset < LIZARD_MAX_16BIT_OFFSET) { /* Encode Literal length */ if (litLength >= MAX_SHORT_LITLEN) { size_t len = litLength - MAX_SHORT_LITLEN; @@ -233,22 +233,22 @@ FORCE_INLINE size_t LIZ_get_price_LZ5v2(LIZ_stream_t* const ctx, int rep, const } else token = (BYTE)litLength; - if (offset >= LIZ_MAX_16BIT_OFFSET) { + if (offset >= LIZARD_MAX_16BIT_OFFSET) { token+=(1<huffType && ctx->params.parserType != LIZ_parser_lowestPrice) - price += LIZ_GET_TOKEN_PRICE_LZ5v2(token); + if (ctx->huffType && ctx->params.parserType != Lizard_parser_lowestPrice) + price += LIZARD_GET_TOKEN_PRICE_LIZv1(token); else price += 8; } } /* Encode Offset */ - if (offset >= LIZ_MAX_16BIT_OFFSET) { // 24-bit offset - if (matchLength < MM_LONGOFF) return LIZ_MAX_PRICE; // error + if (offset >= LIZARD_MAX_16BIT_OFFSET) { // 24-bit offset + if (matchLength < MM_LONGOFF) return LIZARD_MAX_PRICE; // error - if (matchLength - MM_LONGOFF >= LIZ_LAST_LONG_OFF) { - size_t len = matchLength - MM_LONGOFF - LIZ_LAST_LONG_OFF; - token = LIZ_LAST_LONG_OFF; + if (matchLength - MM_LONGOFF >= LIZARD_LAST_LONG_OFF) { + size_t len = matchLength - MM_LONGOFF - LIZARD_LAST_LONG_OFF; + token = LIZARD_LAST_LONG_OFF; if (len >= (1<<16)) price += 32; else if (len >= 254) price += 24; else price += 8; @@ -262,25 +262,25 @@ FORCE_INLINE size_t LIZ_get_price_LZ5v2(LIZ_stream_t* const ctx, int rep, const if (offset == 0) { token+=(1<= MAX_SHORT_MATCHLEN) { - token += (BYTE)(MAX_SHORT_MATCHLEN<= (1<<16)) price += 32; else if (length >= 254) price += 24; else price += 8; } - else token += (BYTE)(length< 0 || matchLength > 0) { - int offset_load = LIZ_highbit32(offset); + int offset_load = Lizard_highbit32(offset); if (ctx->huffType) { price += ((offset_load>=20) ? ((offset_load-19)*4) : 0); price += 4 + (matchLength==1); @@ -288,13 +288,13 @@ FORCE_INLINE size_t LIZ_get_price_LZ5v2(LIZ_stream_t* const ctx, int rep, const price += ((offset_load>=16) ? ((offset_load-15)*4) : 0); price += 6 + (matchLength==1); } - if (ctx->huffType && ctx->params.parserType != LIZ_parser_lowestPrice) - price += LIZ_GET_TOKEN_PRICE_LZ5v2(token); + if (ctx->huffType && ctx->params.parserType != Lizard_parser_lowestPrice) + price += LIZARD_GET_TOKEN_PRICE_LIZv1(token); else price += 8; } else { - if (ctx->huffType && ctx->params.parserType != LIZ_parser_lowestPrice) - price += LIZ_GET_TOKEN_PRICE_LZ5v2(token); // 1=better ratio + if (ctx->huffType && ctx->params.parserType != Lizard_parser_lowestPrice) + price += LIZARD_GET_TOKEN_PRICE_LIZv1(token); // 1=better ratio } return price; diff --git a/C/lizard/liz_compress_lz4.h b/C/lizard/lizard_compress_lz4.h similarity index 70% rename from C/lizard/liz_compress_lz4.h rename to C/lizard/lizard_compress_lz4.h index 5cd725ad..00686b68 100644 --- a/C/lizard/liz_compress_lz4.h +++ b/C/lizard/lizard_compress_lz4.h @@ -1,7 +1,7 @@ -#define LIZ_LENGTH_SIZE_LZ4(len) ((len >= (1<<16)+RUN_MASK_LZ4) ? 5 : ((len >= 254+RUN_MASK_LZ4) ? 3 : ((len >= RUN_MASK_LZ4) ? 1 : 0))) +#define LIZARD_LENGTH_SIZE_LZ4(len) ((len >= (1<<16)+RUN_MASK_LZ4) ? 5 : ((len >= 254+RUN_MASK_LZ4) ? 3 : ((len >= RUN_MASK_LZ4) ? 1 : 0))) -FORCE_INLINE int LIZ_encodeSequence_LZ4 ( - LIZ_stream_t* ctx, +FORCE_INLINE int Lizard_encodeSequence_LZ4 ( + Lizard_stream_t* ctx, const BYTE** ip, const BYTE** anchor, size_t matchLength, @@ -14,7 +14,7 @@ FORCE_INLINE int LIZ_encodeSequence_LZ4 ( COMPLOG_CODEWORDS_LZ4("literal : %u -- match : %u -- offset : %u\n", (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match)); /* Encode Literal length */ - // if (ctx->literalsPtr > ctx->literalsEnd - length - LIZ_LENGTH_SIZE_LZ4(length) - 2 - WILDCOPYLENGTH) { LIZ_LOG_COMPRESS_LZ4("encodeSequence overflow1\n"); return 1; } /* Check output limit */ + // if (ctx->literalsPtr > ctx->literalsEnd - length - LIZARD_LENGTH_SIZE_LZ4(length) - 2 - WILDCOPYLENGTH) { LIZARD_LOG_COMPRESS_LZ4("encodeSequence overflow1\n"); return 1; } /* Check output limit */ if (length >= RUN_MASK_LZ4) { size_t len = length - RUN_MASK_LZ4; *token = RUN_MASK_LZ4; @@ -26,13 +26,13 @@ FORCE_INLINE int LIZ_encodeSequence_LZ4 ( /* Copy Literals */ if (length > 0) { - LIZ_wildCopy(ctx->literalsPtr, *anchor, (ctx->literalsPtr) + length); -#if 0 //def LIZ_USE_HUFFMAN + Lizard_wildCopy(ctx->literalsPtr, *anchor, (ctx->literalsPtr) + length); +#if 0 //def LIZARD_USE_HUFFMAN ctx->litSum += (U32)length; ctx->litPriceSum += (U32)(length * ctx->log2LitSum); { U32 u; for (u=0; u < length; u++) { - ctx->litPriceSum -= LIZ_highbit32(ctx->litFreq[ctx->literalsPtr[u]]+1); + ctx->litPriceSum -= Lizard_highbit32(ctx->litFreq[ctx->literalsPtr[u]]+1); ctx->litFreq[ctx->literalsPtr[u]]++; } } #endif @@ -40,14 +40,12 @@ FORCE_INLINE int LIZ_encodeSequence_LZ4 ( } /* Encode Offset */ -// if (match > *ip) printf("match > *ip\n"), exit(1); -// if ((U32)(*ip-match) >= (1<<16)) printf("off=%d\n", (U32)(*ip-match)), exit(1); MEM_writeLE16(ctx->literalsPtr, (U16)(*ip-match)); ctx->literalsPtr+=2; /* Encode MatchLength */ length = matchLength - MINMATCH; - // if (ctx->literalsPtr > ctx->literalsEnd - 5 /*LIZ_LENGTH_SIZE_LZ4(length)*/) { LIZ_LOG_COMPRESS_LZ4("encodeSequence overflow2\n"); return 1; } /* Check output limit */ + // if (ctx->literalsPtr > ctx->literalsEnd - 5 /*LIZARD_LENGTH_SIZE_LZ4(length)*/) { LIZARD_LOG_COMPRESS_LZ4("encodeSequence overflow2\n"); return 1; } /* Check output limit */ if (length >= ML_MASK_LZ4) { *token += (BYTE)(ML_MASK_LZ4<huffType) { ctx->flagFreq[*token]++; ctx->flagSum++; - LIZ_setLog2Prices(ctx); + Lizard_setLog2Prices(ctx); } #endif @@ -73,8 +71,8 @@ FORCE_INLINE int LIZ_encodeSequence_LZ4 ( } -FORCE_INLINE int LIZ_encodeLastLiterals_LZ4 ( - LIZ_stream_t* ctx, +FORCE_INLINE int Lizard_encodeLastLiterals_LZ4 ( + Lizard_stream_t* ctx, const BYTE** ip, const BYTE** anchor) { @@ -88,29 +86,28 @@ FORCE_INLINE int LIZ_encodeLastLiterals_LZ4 ( } -#define LIZ_GET_TOKEN_PRICE_LZ4(token) (ctx->log2FlagSum - LIZ_highbit32(ctx->flagFreq[token]+1)) +#define LIZARD_GET_TOKEN_PRICE_LZ4(token) (ctx->log2FlagSum - Lizard_highbit32(ctx->flagFreq[token]+1)) -FORCE_INLINE size_t LIZ_get_price_LZ4(LIZ_stream_t* const ctx, const BYTE *ip, const size_t litLength, U32 offset, size_t matchLength) +FORCE_INLINE size_t Lizard_get_price_LZ4(Lizard_stream_t* const ctx, const BYTE *ip, const size_t litLength, U32 offset, size_t matchLength) { size_t price = 0; BYTE token = 0; -#if 0 //def LIZ_USE_HUFFMAN +#if 0 //def LIZARD_USE_HUFFMAN const BYTE* literals = ip - litLength; U32 u; if (ctx->cachedLiterals == literals && litLength >= ctx->cachedLitLength) { size_t const additional = litLength - ctx->cachedLitLength; - // printf("%d ", (int)litLength - (int)ctx->cachedLitLength); const BYTE* literals2 = ctx->cachedLiterals + ctx->cachedLitLength; price = ctx->cachedPrice + additional * ctx->log2LitSum; for (u=0; u < additional; u++) - price -= LIZ_highbit32(ctx->litFreq[literals2[u]]+1); + price -= Lizard_highbit32(ctx->litFreq[literals2[u]]+1); ctx->cachedPrice = (U32)price; ctx->cachedLitLength = (U32)litLength; } else { price = litLength * ctx->log2LitSum; for (u=0; u < litLength; u++) - price -= LIZ_highbit32(ctx->litFreq[literals[u]]+1); + price -= Lizard_highbit32(ctx->litFreq[literals[u]]+1); if (litLength >= 12) { ctx->cachedLiterals = literals; @@ -140,8 +137,8 @@ FORCE_INLINE size_t LIZ_get_price_LZ4(LIZ_stream_t* const ctx, const BYTE *ip, c size_t length; price += 16; /* Encode Offset */ - if (offset < 8) return LIZ_MAX_PRICE; // error - if (matchLength < MINMATCH) return LIZ_MAX_PRICE; // error + if (offset < 8) return LIZARD_MAX_PRICE; // error + if (matchLength < MINMATCH) return LIZARD_MAX_PRICE; // error length = matchLength - MINMATCH; if (length >= ML_MASK_LZ4) { @@ -156,7 +153,7 @@ FORCE_INLINE size_t LIZ_get_price_LZ4(LIZ_stream_t* const ctx, const BYTE *ip, c if (ctx->huffType) { if (offset > 0 || matchLength > 0) price += 2; - price += LIZ_GET_TOKEN_PRICE_LZ4(token); + price += LIZARD_GET_TOKEN_PRICE_LZ4(token); } else { price += 8; // token } diff --git a/C/lizard/lizard_decompress.c b/C/lizard/lizard_decompress.c new file mode 100644 index 00000000..3ae8e057 --- /dev/null +++ b/C/lizard/lizard_decompress.c @@ -0,0 +1,372 @@ +/* + Lizard - Fast LZ compression algorithm + Copyright (C) 2011-2016, Yann Collet + Copyright (C) 2016-2017, Przemyslaw Skibinski + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Lizard source repository : https://github.com/inikep/lizard +*/ + + +/************************************** +* Includes +**************************************/ +//#define LIZARD_STATS 1 // 0=simple stats, 1=more, 2=full +#ifdef LIZARD_STATS + #include "test/lizard_stats.h" +#endif +#include "lizard_compress.h" +#include "lizard_decompress.h" +#include "lizard_common.h" +#include // printf +#include // intptr_t + + +/*-************************************ +* Local Structures and types +**************************************/ +typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive; +typedef enum { full = 0, partial = 1 } earlyEnd_directive; + +#include "lizard_decompress_lz4.h" +#ifndef USE_LZ4_ONLY + #ifdef LIZARD_USE_TEST + #include "test/lizard_common_test.h" + #include "test/lizard_decompress_test.h" + #else + #include "lizard_decompress_liz.h" + #endif +#endif +#include "../zstd/huf.h" + + +/*-***************************** +* Decompression functions +*******************************/ + +FORCE_INLINE size_t Lizard_readStream(int flag, const BYTE** ip, const BYTE* const iend, BYTE* op, BYTE* const oend, const BYTE** streamPtr, const BYTE** streamEnd, int streamFlag) +{ + if (!flag) { + if (*ip > iend - 3) return 0; + *streamPtr = *ip + 3; + *streamEnd = *streamPtr + MEM_readLE24(*ip); + if (*streamEnd < *streamPtr) return 0; + *ip = *streamEnd; +#ifdef LIZARD_STATS + uncompr_stream[streamFlag] += *streamEnd-*streamPtr; +#else + (void)streamFlag; +#endif + return 1; + } else { +#ifndef LIZARD_NO_HUFFMAN + size_t res, streamLen, comprStreamLen; + + if (*ip > iend - 6) return 0; + streamLen = MEM_readLE24(*ip); + comprStreamLen = MEM_readLE24(*ip + 3); + + if ((op > oend - streamLen) || (*ip + comprStreamLen > iend - 6)) return 0; + res = HUF_decompress(op, streamLen, *ip + 6, comprStreamLen); + if (HUF_isError(res) || (res != streamLen)) return 0; + + *ip += comprStreamLen + 6; + *streamPtr = op; + *streamEnd = *streamPtr + streamLen; +#ifdef LIZARD_STATS + compr_stream[streamFlag] += comprStreamLen + 6; + decompr_stream[streamFlag] += *streamEnd-*streamPtr; +#endif + return 1; +#else + fprintf(stderr, "compiled with LIZARD_NO_HUFFMAN\n"); + (void)op; (void)oend; + return 0; +#endif + } +} + + +FORCE_INLINE int Lizard_decompress_generic( + const char* source, + char* const dest, + int inputSize, + int outputSize, /* this value is the max size of Output Buffer. */ + int partialDecoding, /* full, partial */ + int targetOutputSize, /* only used if partialDecoding==partial */ + int dict, /* noDict, withPrefix64k, usingExtDict */ + const BYTE* const lowPrefix, /* == dest if dict == noDict */ + const BYTE* const dictStart, /* only if dict==usingExtDict */ + const size_t dictSize /* note : = 0 if noDict */ + ) +{ + /* Local Variables */ + const BYTE* ip = (const BYTE*) source, *istart = (const BYTE*) source; + const BYTE* const iend = ip + inputSize; + BYTE* op = (BYTE*) dest; + BYTE* const oend = op + outputSize; + BYTE* oexit = op + targetOutputSize; + Lizard_parameters params; + Lizard_dstream_t ctx; + BYTE* decompFlagsBase, *decompOff24Base, *decompOff16Base, *decompLiteralsBase = NULL; + int res, compressionLevel; + + if (inputSize < 1) { LIZARD_LOG_DECOMPRESS("inputSize=%d outputSize=%d targetOutputSize=%d partialDecoding=%d\n", inputSize, outputSize, targetOutputSize, partialDecoding); return 0; } + + compressionLevel = *ip++; + + if (compressionLevel < LIZARD_MIN_CLEVEL || compressionLevel > LIZARD_MAX_CLEVEL) { + LIZARD_LOG_DECOMPRESS("ERROR Lizard_decompress_generic inputSize=%d compressionLevel=%d\n", inputSize, compressionLevel); + return -1; + } + + LIZARD_LOG_DECOMPRESS("Lizard_decompress_generic ip=%p inputSize=%d targetOutputSize=%d dest=%p outputSize=%d cLevel=%d dict=%d dictSize=%d dictStart=%p partialDecoding=%d\n", ip, inputSize, targetOutputSize, dest, outputSize, compressionLevel, dict, (int)dictSize, dictStart, partialDecoding); + + decompLiteralsBase = (BYTE*)malloc(4*LIZARD_HUF_BLOCK_SIZE); + if (!decompLiteralsBase) return -1; + decompFlagsBase = decompLiteralsBase + LIZARD_HUF_BLOCK_SIZE; + decompOff24Base = decompFlagsBase + LIZARD_HUF_BLOCK_SIZE; + decompOff16Base = decompOff24Base + LIZARD_HUF_BLOCK_SIZE; + +#ifdef LIZARD_STATS + init_stats(); +#endif + (void)istart; + + while (ip < iend) + { + res = *ip++; + if (res == LIZARD_FLAG_UNCOMPRESSED) /* uncompressed */ + { + uint32_t length; + if (ip > iend - 3) { LIZARD_LOG_DECOMPRESS("UNCOMPRESSED ip[%p] > iend[%p] - 3\n", ip, iend); goto _output_error; } + length = MEM_readLE24(ip); + ip += 3; + // printf("%d: total=%d block=%d UNCOMPRESSED op=%p oexit=%p oend=%p\n", (int)(op-(BYTE*)dest) ,(int)(ip-istart), length, op, oexit, oend); + if (ip + length > iend || op + length > oend) { LIZARD_LOG_DECOMPRESS("UNCOMPRESSED ip[%p]+length[%d] > iend[%p]\n", ip, length, iend); goto _output_error; } + memcpy(op, ip, length); + op += length; + ip += length; + if ((partialDecoding) && (op >= oexit)) break; +#ifdef LIZARD_STATS + uncompr_stream[LIZARD_STREAM_UNCOMPRESSED] += length; +#endif + continue; + } + + if (res&LIZARD_FLAG_LEN) { + LIZARD_LOG_DECOMPRESS("res=%d\n", res); goto _output_error; + } + + if (ip > iend - 5*3) goto _output_error; + ctx.lenPtr = (const BYTE*)ip + 3; + ctx.lenEnd = ctx.lenPtr + MEM_readLE24(ip); + if (ctx.lenEnd < ctx.lenPtr || (ctx.lenEnd > iend - 3)) goto _output_error; +#ifdef LIZARD_STATS + uncompr_stream[LIZARD_STREAM_LEN] += ctx.lenEnd-ctx.lenPtr + 3; +#endif + ip = ctx.lenEnd; + + { size_t streamLen; +#ifdef LIZARD_USE_LOGS + const BYTE* ipos; + size_t comprFlagsLen, comprLiteralsLen, total; +#endif + streamLen = Lizard_readStream(res&LIZARD_FLAG_OFFSET16, &ip, iend, decompOff16Base, decompOff16Base + LIZARD_HUF_BLOCK_SIZE, &ctx.offset16Ptr, &ctx.offset16End, LIZARD_STREAM_OFFSET16); + if (streamLen == 0) goto _output_error; + + streamLen = Lizard_readStream(res&LIZARD_FLAG_OFFSET24, &ip, iend, decompOff24Base, decompOff24Base + LIZARD_HUF_BLOCK_SIZE, &ctx.offset24Ptr, &ctx.offset24End, LIZARD_STREAM_OFFSET24); + if (streamLen == 0) goto _output_error; + +#ifdef LIZARD_USE_LOGS + ipos = ip; + streamLen = Lizard_readStream(res&LIZARD_FLAG_FLAGS, &ip, iend, decompFlagsBase, decompFlagsBase + LIZARD_HUF_BLOCK_SIZE, &ctx.flagsPtr, &ctx.flagsEnd, LIZARD_STREAM_FLAGS); + if (streamLen == 0) goto _output_error; + streamLen = (size_t)(ctx.flagsEnd-ctx.flagsPtr); + comprFlagsLen = ((size_t)(ip - ipos) + 3 >= streamLen) ? 0 : (size_t)(ip - ipos); + ipos = ip; +#else + streamLen = Lizard_readStream(res&LIZARD_FLAG_FLAGS, &ip, iend, decompFlagsBase, decompFlagsBase + LIZARD_HUF_BLOCK_SIZE, &ctx.flagsPtr, &ctx.flagsEnd, LIZARD_STREAM_FLAGS); + if (streamLen == 0) goto _output_error; +#endif + + streamLen = Lizard_readStream(res&LIZARD_FLAG_LITERALS, &ip, iend, decompLiteralsBase, decompLiteralsBase + LIZARD_HUF_BLOCK_SIZE, &ctx.literalsPtr, &ctx.literalsEnd, LIZARD_STREAM_LITERALS); + if (streamLen == 0) goto _output_error; +#ifdef LIZARD_USE_LOGS + streamLen = (size_t)(ctx.literalsEnd-ctx.literalsPtr); + comprLiteralsLen = ((size_t)(ip - ipos) + 3 >= streamLen) ? 0 : (size_t)(ip - ipos); + total = (size_t)(ip-(ctx.lenEnd-1)); +#endif + + if (ip > iend) goto _output_error; + + LIZARD_LOG_DECOMPRESS("%d: total=%d block=%d flagsLen=%d(HUF=%d) literalsLen=%d(HUF=%d) offset16Len=%d offset24Len=%d lengthsLen=%d \n", (int)(op-(BYTE*)dest) ,(int)(ip-istart), (int)total, + (int)(ctx.flagsEnd-ctx.flagsPtr), (int)comprFlagsLen, (int)(ctx.literalsEnd-ctx.literalsPtr), (int)comprLiteralsLen, + (int)(ctx.offset16End-ctx.offset16Ptr), (int)(ctx.offset24End-ctx.offset24Ptr), (int)(ctx.lenEnd-ctx.lenPtr)); + } + + ctx.last_off = -LIZARD_INIT_LAST_OFFSET; + params = Lizard_defaultParameters[compressionLevel - LIZARD_MIN_CLEVEL]; + if (params.decompressType == Lizard_coderwords_LZ4) + res = Lizard_decompress_LZ4(&ctx, op, outputSize, partialDecoding, targetOutputSize, dict, lowPrefix, dictStart, dictSize, compressionLevel); + else +#ifdef USE_LZ4_ONLY + res = Lizard_decompress_LZ4(&ctx, op, outputSize, partialDecoding, targetOutputSize, dict, lowPrefix, dictStart, dictSize, compressionLevel); +#else + res = Lizard_decompress_LIZv1(&ctx, op, outputSize, partialDecoding, targetOutputSize, dict, lowPrefix, dictStart, dictSize, compressionLevel); +#endif + LIZARD_LOG_DECOMPRESS("Lizard_decompress_generic res=%d inputSize=%d\n", res, (int)(ctx.literalsEnd-ctx.lenEnd)); + + if (res <= 0) { free(decompLiteralsBase); return res; } + + op += res; + outputSize -= res; + if ((partialDecoding) && (op >= oexit)) break; + } + +#ifdef LIZARD_STATS + print_stats(); +#endif + + LIZARD_LOG_DECOMPRESS("Lizard_decompress_generic total=%d\n", (int)(op-(BYTE*)dest)); + free(decompLiteralsBase); + return (int)(op-(BYTE*)dest); + +_output_error: + LIZARD_LOG_DECOMPRESS("Lizard_decompress_generic ERROR ip=%p iend=%p\n", ip, iend); + free(decompLiteralsBase); + return -1; +} + + +int Lizard_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) +{ + return Lizard_decompress_generic(source, dest, compressedSize, maxDecompressedSize, full, 0, noDict, (BYTE*)dest, NULL, 0); +} + +int Lizard_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize) +{ + return Lizard_decompress_generic(source, dest, compressedSize, maxDecompressedSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0); +} + + +/*===== streaming decompression functions =====*/ + + +/* + * If you prefer dynamic allocation methods, + * Lizard_createStreamDecode() + * provides a pointer (void*) towards an initialized Lizard_streamDecode_t structure. + */ +Lizard_streamDecode_t* Lizard_createStreamDecode(void) +{ + Lizard_streamDecode_t* lizards = (Lizard_streamDecode_t*) ALLOCATOR(1, sizeof(Lizard_streamDecode_t)); + return lizards; +} + +int Lizard_freeStreamDecode (Lizard_streamDecode_t* Lizard_stream) +{ + FREEMEM(Lizard_stream); + return 0; +} + +/*! + * Lizard_setStreamDecode() : + * Use this function to instruct where to find the dictionary. + * This function is not necessary if previous data is still available where it was decoded. + * Loading a size of 0 is allowed (same effect as no dictionary). + * Return : 1 if OK, 0 if error + */ +int Lizard_setStreamDecode (Lizard_streamDecode_t* Lizard_streamDecode, const char* dictionary, int dictSize) +{ + Lizard_streamDecode_t* lizardsd = (Lizard_streamDecode_t*) Lizard_streamDecode; + lizardsd->prefixSize = (size_t) dictSize; + lizardsd->prefixEnd = (const BYTE*) dictionary + dictSize; + lizardsd->externalDict = NULL; + lizardsd->extDictSize = 0; + return 1; +} + +/* +*_continue() : + These decoding functions allow decompression of multiple blocks in "streaming" mode. + Previously decoded blocks must still be available at the memory position where they were decoded. + If it's not possible, save the relevant part of decoded data into a safe buffer, + and indicate where it stands using Lizard_setStreamDecode() +*/ +int Lizard_decompress_safe_continue (Lizard_streamDecode_t* Lizard_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + Lizard_streamDecode_t* lizardsd = (Lizard_streamDecode_t*) Lizard_streamDecode; + int result; + + if (lizardsd->prefixEnd == (BYTE*)dest) { + result = Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize, + full, 0, usingExtDict, lizardsd->prefixEnd - lizardsd->prefixSize, lizardsd->externalDict, lizardsd->extDictSize); + if (result <= 0) return result; + lizardsd->prefixSize += result; + lizardsd->prefixEnd += result; + } else { + lizardsd->extDictSize = lizardsd->prefixSize; + lizardsd->externalDict = lizardsd->prefixEnd - lizardsd->extDictSize; + result = Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize, + full, 0, usingExtDict, (BYTE*)dest, lizardsd->externalDict, lizardsd->extDictSize); + if (result <= 0) return result; + lizardsd->prefixSize = result; + lizardsd->prefixEnd = (BYTE*)dest + result; + } + + return result; +} + + +/* +Advanced decoding functions : +*_usingDict() : + These decoding functions work the same as "_continue" ones, + the dictionary must be explicitly provided within parameters +*/ + +int Lizard_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) +{ + if (dictSize==0) + return Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, noDict, (BYTE*)dest, NULL, 0); + if (dictStart+dictSize == dest) + { + if (dictSize >= (int)(LIZARD_DICT_SIZE - 1)) + return Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, withPrefix64k, (BYTE*)dest-LIZARD_DICT_SIZE, NULL, 0); + return Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); + } + return Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +/* debug function */ +int Lizard_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) +{ + return Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + diff --git a/C/lizard/liz_decompress.h b/C/lizard/lizard_decompress.h similarity index 63% rename from C/lizard/liz_decompress.h rename to C/lizard/lizard_decompress.h index 1d5e2de4..a54ce048 100644 --- a/C/lizard/liz_decompress.h +++ b/C/lizard/lizard_decompress.h @@ -1,8 +1,8 @@ /* - LZ5 - Fast LZ compression algorithm + Lizard - Fast LZ compression algorithm Header File - Copyright (C) 2011-2016, Yann Collet. - Copyright (C) 2016, Przemyslaw Skibinski + Copyright (C) 2011-2016, Yann Collet + Copyright (C) 2016-2017, Przemyslaw Skibinski BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) @@ -30,31 +30,31 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - LZ5 source repository : https://github.com/inikep/lz5 + - Lizard source repository : https://github.com/inikep/lizard */ -#ifndef LIZ_DECOMPRESS_H_2983 -#define LIZ_DECOMPRESS_H_2983 +#ifndef LIZARD_DECOMPRESS_H_2983 +#define LIZARD_DECOMPRESS_H_2983 #if defined (__cplusplus) extern "C" { #endif -#include "mem.h" /* U32 */ +#include "../zstd/mem.h" /* U32 */ /*^*************************************************************** * Export parameters *****************************************************************/ /* -* LIZ_DLL_EXPORT : +* LIZARD_DLL_EXPORT : * Enable exporting of functions when building a Windows DLL */ -#if defined(LIZ_DLL_EXPORT) && (LIZ_DLL_EXPORT==1) -# define LZ5DLIB_API __declspec(dllexport) -#elif defined(LIZ_DLL_IMPORT) && (LIZ_DLL_IMPORT==1) -# define LZ5DLIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#if defined(LIZARD_DLL_EXPORT) && (LIZARD_DLL_EXPORT==1) +# define LIZARDDLIB_API __declspec(dllexport) +#elif defined(LIZARD_DLL_IMPORT) && (LIZARD_DLL_IMPORT==1) +# define LIZARDDLIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ #else -# define LZ5DLIB_API +# define LIZARDDLIB_API #endif @@ -63,7 +63,7 @@ extern "C" { **************************************/ /* -LIZ_decompress_safe() : +Lizard_decompress_safe() : compressedSize : is the precise full size of the compressed block. maxDecompressedSize : is the size of destination buffer, which must be already allocated. return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize) @@ -72,12 +72,12 @@ LIZ_decompress_safe() : This function is protected against buffer overflow exploits, including malicious data packets. It never writes outside output buffer, nor reads outside input buffer. */ -LZ5DLIB_API int LIZ_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); +LIZARDDLIB_API int Lizard_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); /*! -LIZ_decompress_safe_partial() : +Lizard_decompress_safe_partial() : This function decompress a compressed block of size 'compressedSize' at position 'source' into destination buffer 'dest' of size 'maxDecompressedSize'. The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, @@ -88,7 +88,7 @@ LIZ_decompress_safe_partial() : If the source stream is detected malformed, the function will stop decoding and return a negative result. This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets */ -LZ5DLIB_API int LIZ_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); +LIZARDDLIB_API int Lizard_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); @@ -100,60 +100,60 @@ typedef struct { size_t extDictSize; const BYTE* prefixEnd; size_t prefixSize; -} LIZ_streamDecode_t; +} Lizard_streamDecode_t; /* - * LIZ_streamDecode_t - * information structure to track an LZ5 stream. - * init this structure content using LIZ_setStreamDecode or memset() before first use ! + * Lizard_streamDecode_t + * information structure to track an Lizard stream. + * init this structure content using Lizard_setStreamDecode or memset() before first use ! * - * In the context of a DLL (liblz5) please prefer usage of construction methods below. - * They are more future proof, in case of a change of LIZ_streamDecode_t size in the future. - * LIZ_createStreamDecode will allocate and initialize an LIZ_streamDecode_t structure - * LIZ_freeStreamDecode releases its memory. + * In the context of a DLL (liblizard) please prefer usage of construction methods below. + * They are more future proof, in case of a change of Lizard_streamDecode_t size in the future. + * Lizard_createStreamDecode will allocate and initialize an Lizard_streamDecode_t structure + * Lizard_freeStreamDecode releases its memory. */ -LZ5DLIB_API LIZ_streamDecode_t* LIZ_createStreamDecode(void); -LZ5DLIB_API int LIZ_freeStreamDecode (LIZ_streamDecode_t* LIZ_stream); +LIZARDDLIB_API Lizard_streamDecode_t* Lizard_createStreamDecode(void); +LIZARDDLIB_API int Lizard_freeStreamDecode (Lizard_streamDecode_t* Lizard_stream); -/*! LIZ_setStreamDecode() : +/*! Lizard_setStreamDecode() : * Use this function to instruct where to find the dictionary. * Setting a size of 0 is allowed (same effect as reset). * @return : 1 if OK, 0 if error */ -LZ5DLIB_API int LIZ_setStreamDecode (LIZ_streamDecode_t* LIZ_streamDecode, const char* dictionary, int dictSize); +LIZARDDLIB_API int Lizard_setStreamDecode (Lizard_streamDecode_t* Lizard_streamDecode, const char* dictionary, int dictSize); /* *_continue() : These decoding functions allow decompression of multiple blocks in "streaming" mode. - Previously decoded blocks *must* remain available at the memory position where they were decoded (up to LIZ_DICT_SIZE) + Previously decoded blocks *must* remain available at the memory position where they were decoded (up to LIZARD_DICT_SIZE) In the case of a ring buffers, decoding buffer must be either : - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) - In which case, the decoding & encoding ring buffer can have any size, including small ones ( < LIZ_DICT_SIZE). + In which case, the decoding & encoding ring buffer can have any size, including small ones ( < LIZARD_DICT_SIZE). - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block. In which case, encoding and decoding buffers do not need to be synchronized, - and encoding ring buffer can have any size, including small ones ( < LIZ_DICT_SIZE). - - _At least_ LIZ_DICT_SIZE + 8 bytes + maxBlockSize. + and encoding ring buffer can have any size, including small ones ( < LIZARD_DICT_SIZE). + - _At least_ LIZARD_DICT_SIZE + 8 bytes + maxBlockSize. In which case, encoding and decoding buffers do not need to be synchronized, and encoding ring buffer can have any size, including larger than decoding buffer. - Whenever these conditions are not possible, save the last LIZ_DICT_SIZE of decoded data into a safe buffer, - and indicate where it is saved using LIZ_setStreamDecode() + Whenever these conditions are not possible, save the last LIZARD_DICT_SIZE of decoded data into a safe buffer, + and indicate where it is saved using Lizard_setStreamDecode() */ -LZ5DLIB_API int LIZ_decompress_safe_continue (LIZ_streamDecode_t* LIZ_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); +LIZARDDLIB_API int Lizard_decompress_safe_continue (Lizard_streamDecode_t* Lizard_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); /* Advanced decoding functions : *_usingDict() : These decoding functions work the same as - a combination of LIZ_setStreamDecode() followed by LIZ_decompress_x_continue() - They are stand-alone. They don't need nor update an LIZ_streamDecode_t structure. + a combination of Lizard_setStreamDecode() followed by Lizard_decompress_x_continue() + They are stand-alone. They don't need nor update an Lizard_streamDecode_t structure. */ -LZ5DLIB_API int LIZ_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); +LIZARDDLIB_API int Lizard_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); #if defined (__cplusplus) } #endif -#endif /* LIZ_DECOMPRESS_H_2983827168210 */ +#endif /* LIZARD_DECOMPRESS_H_2983827168210 */ diff --git a/C/lizard/liz_decompress_lz5v2.h b/C/lizard/lizard_decompress_liz.h similarity index 72% rename from C/lizard/liz_decompress_lz5v2.h rename to C/lizard/lizard_decompress_liz.h index 4a693880..d306a7cb 100644 --- a/C/lizard/liz_decompress_lz5v2.h +++ b/C/lizard/lizard_decompress_liz.h @@ -5,14 +5,14 @@ flag 0-30 - 24-bit offset, 31 match lengths (16-46), no literal length */ -/*! LIZ_decompress_LZ5v2() : +/*! Lizard_decompress_LIZv1() : * This generic decompression function cover all use cases. * It shall be instantiated several times, using different sets of directives * Note that it is important this generic function is really inlined, * in order to remove useless branches during compilation optimization. */ -FORCE_INLINE int LIZ_decompress_LZ5v2( - LIZ_dstream_t* ctx, +FORCE_INLINE int Lizard_decompress_LIZv1( + Lizard_dstream_t* ctx, BYTE* const dest, int outputSize, /* this value is the max size of Output Buffer. */ @@ -37,12 +37,11 @@ FORCE_INLINE int LIZ_decompress_LZ5v2( const BYTE* const lowLimit = lowPrefix - dictSize; const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; - const int checkOffset = (dictSize < (int)(LIZ_DICT_SIZE)); + const int checkOffset = (dictSize < (int)(LIZARD_DICT_SIZE)); intptr_t last_off = ctx->last_off; intptr_t length = 0; (void)compressionLevel; - (void)LIZ_wildCopy; /* Special cases */ if (unlikely(outputSize==0)) return ((inputSize==1) && (*ctx->flagsPtr==0)) ? 0 : -1; /* Empty output buffer */ @@ -58,11 +57,10 @@ FORCE_INLINE int LIZ_decompress_LZ5v2( /* get literal length */ token = *ctx->flagsPtr++; - // LIZ_LOG_DECOMPRESS_LZ5v2("token : %u\n", (U32)token); if (token >= 32) { if ((length=(token & MAX_SHORT_LITLEN)) == MAX_SHORT_LITLEN) { - if (unlikely(ctx->literalsPtr > iend - 1)) { LIZ_LOG_DECOMPRESS_LZ5v2("1"); goto _output_error; } + if (unlikely(ctx->literalsPtr > iend - 1)) { LIZARD_LOG_DECOMPRESS_LIZv1("1"); goto _output_error; } length = *ctx->literalsPtr; if unlikely(length >= 254) { if (length == 254) { @@ -75,28 +73,28 @@ FORCE_INLINE int LIZ_decompress_LZ5v2( } length += MAX_SHORT_LITLEN; ctx->literalsPtr++; - if (unlikely((size_t)(op+length)<(size_t)(op))) { LIZ_LOG_DECOMPRESS_LZ5v2("2"); goto _output_error; } /* overflow detection */ - if (unlikely((size_t)(ctx->literalsPtr+length)<(size_t)(ctx->literalsPtr))) { LIZ_LOG_DECOMPRESS_LZ5v2("3"); goto _output_error; } /* overflow detection */ + if (unlikely((size_t)(op+length)<(size_t)(op))) { LIZARD_LOG_DECOMPRESS_LIZv1("2"); goto _output_error; } /* overflow detection */ + if (unlikely((size_t)(ctx->literalsPtr+length)<(size_t)(ctx->literalsPtr))) { LIZARD_LOG_DECOMPRESS_LIZv1("3"); goto _output_error; } /* overflow detection */ } /* copy literals */ cpy = op + length; - if (unlikely(cpy > oend - WILDCOPYLENGTH || ctx->literalsPtr > iend - WILDCOPYLENGTH)) { LIZ_LOG_DECOMPRESS_LZ5v2("offset outside buffers\n"); goto _output_error; } /* Error : offset outside buffers */ + if (unlikely(cpy > oend - WILDCOPYLENGTH || ctx->literalsPtr > iend - WILDCOPYLENGTH)) { LIZARD_LOG_DECOMPRESS_LIZv1("offset outside buffers\n"); goto _output_error; } /* Error : offset outside buffers */ #if 1 - LIZ_wildCopy16(op, ctx->literalsPtr, cpy); + Lizard_wildCopy16(op, ctx->literalsPtr, cpy); op = cpy; ctx->literalsPtr += length; #else - LIZ_copy8(op, ctx->literalsPtr); - LIZ_copy8(op+8, ctx->literalsPtr+8); + Lizard_copy8(op, ctx->literalsPtr); + Lizard_copy8(op+8, ctx->literalsPtr+8); if (length > 16) - LIZ_wildCopy16(op + 16, ctx->literalsPtr + 16, cpy); + Lizard_wildCopy16(op + 16, ctx->literalsPtr + 16, cpy); op = cpy; ctx->literalsPtr += length; #endif /* get offset */ - if (unlikely(ctx->offset16Ptr > ctx->offset16End)) { LIZ_LOG_DECOMPRESS_LZ5v2("(ctx->offset16Ptr > ctx->offset16End\n"); goto _output_error; } + if (unlikely(ctx->offset16Ptr > ctx->offset16End)) { LIZARD_LOG_DECOMPRESS_LIZv1("(ctx->offset16Ptr > ctx->offset16End\n"); goto _output_error; } #if 1 { /* branchless */ intptr_t new_off = MEM_readLE16(ctx->offset16Ptr); @@ -105,19 +103,17 @@ FORCE_INLINE int LIZ_decompress_LZ5v2( ctx->offset16Ptr = (BYTE*)((uintptr_t)ctx->offset16Ptr + (not_repCode & 2)); } #else - if ((token >> ML_RUN_BITS_LZ5v2) == 0) + if ((token >> ML_RUN_BITS_LIZv1) == 0) { last_off = -(intptr_t)MEM_readLE16(ctx->offset16Ptr); ctx->offset16Ptr += 2; - // LZ5v2_DEBUG("MEM_readLE16 offset=%d\n", (int)offset); } #endif /* get matchlength */ - length = (token >> RUN_BITS_LZ5v2) & MAX_SHORT_MATCHLEN; - // printf("length=%d token=%d\n", (int)length, (int)token); + length = (token >> RUN_BITS_LIZv1) & MAX_SHORT_MATCHLEN; if (length == MAX_SHORT_MATCHLEN) { - if (unlikely(ctx->literalsPtr > iend - 1)) { LIZ_LOG_DECOMPRESS_LZ5v2("6"); goto _output_error; } + if (unlikely(ctx->literalsPtr > iend - 1)) { LIZARD_LOG_DECOMPRESS_LIZv1("6"); goto _output_error; } length = *ctx->literalsPtr; if unlikely(length >= 254) { if (length == 254) { @@ -130,23 +126,23 @@ FORCE_INLINE int LIZ_decompress_LZ5v2( } length += MAX_SHORT_MATCHLEN; ctx->literalsPtr++; - if (unlikely((size_t)(op+length)<(size_t)(op))) { LIZ_LOG_DECOMPRESS_LZ5v2("7"); goto _output_error; } /* overflow detection */ + if (unlikely((size_t)(op+length)<(size_t)(op))) { LIZARD_LOG_DECOMPRESS_LIZv1("7"); goto _output_error; } /* overflow detection */ } - DECOMPLOG_CODEWORDS_LZ5v2("T32+ literal=%u match=%u offset=%d ipos=%d opos=%d\n", (U32)litLength, (U32)length, (int)-last_off, (U32)(ctx->flagsPtr-blockBase), (U32)(op-dest)); + DECOMPLOG_CODEWORDS_LIZv1("T32+ literal=%u match=%u offset=%d ipos=%d opos=%d\n", (U32)litLength, (U32)length, (int)-last_off, (U32)(ctx->flagsPtr-blockBase), (U32)(op-dest)); } else - if (token < LIZ_LAST_LONG_OFF) + if (token < LIZARD_LAST_LONG_OFF) { - if (unlikely(ctx->offset24Ptr > ctx->offset24End - 3)) { LIZ_LOG_DECOMPRESS_LZ5v2("8"); goto _output_error; } + if (unlikely(ctx->offset24Ptr > ctx->offset24End - 3)) { LIZARD_LOG_DECOMPRESS_LIZv1("8"); goto _output_error; } length = token + MM_LONGOFF; last_off = -(intptr_t)MEM_readLE24(ctx->offset24Ptr); ctx->offset24Ptr += 3; - DECOMPLOG_CODEWORDS_LZ5v2("T0-30 literal=%u match=%u offset=%d\n", 0, (U32)length, (int)-last_off); + DECOMPLOG_CODEWORDS_LIZv1("T0-30 literal=%u match=%u offset=%d\n", 0, (U32)length, (int)-last_off); } else { - if (unlikely(ctx->literalsPtr > iend - 1)) { LIZ_LOG_DECOMPRESS_LZ5v2("9"); goto _output_error; } + if (unlikely(ctx->literalsPtr > iend - 1)) { LIZARD_LOG_DECOMPRESS_LIZv1("9"); goto _output_error; } length = *ctx->literalsPtr; if unlikely(length >= 254) { if (length == 254) { @@ -158,20 +154,20 @@ FORCE_INLINE int LIZ_decompress_LZ5v2( } } ctx->literalsPtr++; - length += LIZ_LAST_LONG_OFF + MM_LONGOFF; + length += LIZARD_LAST_LONG_OFF + MM_LONGOFF; - if (unlikely(ctx->offset24Ptr > ctx->offset24End - 3)) { LIZ_LOG_DECOMPRESS_LZ5v2("10"); goto _output_error; } + if (unlikely(ctx->offset24Ptr > ctx->offset24End - 3)) { LIZARD_LOG_DECOMPRESS_LIZv1("10"); goto _output_error; } last_off = -(intptr_t)MEM_readLE24(ctx->offset24Ptr); ctx->offset24Ptr += 3; } match = op + last_off; - if ((checkOffset) && ((unlikely((uintptr_t)(-last_off) > (uintptr_t)op) || (match < lowLimit)))) { LIZ_LOG_DECOMPRESS_LZ5v2("lowPrefix[%p]-dictSize[%d]=lowLimit[%p] match[%p]=op[%p]-last_off[%d]\n", lowPrefix, (int)dictSize, lowLimit, match, op, (int)last_off); goto _output_error; } /* Error : offset outside buffers */ + if ((checkOffset) && ((unlikely((uintptr_t)(-last_off) > (uintptr_t)op) || (match < lowLimit)))) { LIZARD_LOG_DECOMPRESS_LIZv1("lowPrefix[%p]-dictSize[%d]=lowLimit[%p] match[%p]=op[%p]-last_off[%d]\n", lowPrefix, (int)dictSize, lowLimit, match, op, (int)last_off); goto _output_error; } /* Error : offset outside buffers */ /* check external dictionary */ if ((dict==usingExtDict) && (match < lowPrefix)) { - if (unlikely(op + length > oend - WILDCOPYLENGTH)) { LIZ_LOG_DECOMPRESS_LZ5v2("12"); goto _output_error; } /* doesn't respect parsing restriction */ + if (unlikely(op + length > oend - WILDCOPYLENGTH)) { LIZARD_LOG_DECOMPRESS_LIZv1("12"); goto _output_error; } /* doesn't respect parsing restriction */ if (length <= (intptr_t)(lowPrefix - match)) { /* match can be copied as a single segment from external dictionary */ @@ -196,18 +192,18 @@ FORCE_INLINE int LIZ_decompress_LZ5v2( /* copy match within block */ cpy = op + length; - if (unlikely(cpy > oend - WILDCOPYLENGTH)) { LIZ_LOG_DECOMPRESS_LZ5v2("13match=%p lowLimit=%p\n", match, lowLimit); goto _output_error; } /* Error : offset outside buffers */ - LIZ_copy8(op, match); - LIZ_copy8(op+8, match+8); + if (unlikely(cpy > oend - WILDCOPYLENGTH)) { LIZARD_LOG_DECOMPRESS_LIZv1("13match=%p lowLimit=%p\n", match, lowLimit); goto _output_error; } /* Error : offset outside buffers */ + Lizard_copy8(op, match); + Lizard_copy8(op+8, match+8); if (length > 16) - LIZ_wildCopy16(op + 16, match + 16, cpy); + Lizard_wildCopy16(op + 16, match + 16, cpy); op = cpy; } /* last literals */ length = ctx->literalsEnd - ctx->literalsPtr; cpy = op + length; - if ((ctx->literalsPtr+length != iend) || (cpy > oend)) { LIZ_LOG_DECOMPRESS_LZ5v2("14"); goto _output_error; } /* Error : input must be consumed */ + if ((ctx->literalsPtr+length != iend) || (cpy > oend)) { LIZARD_LOG_DECOMPRESS_LIZv1("14"); goto _output_error; } /* Error : input must be consumed */ memcpy(op, ctx->literalsPtr, length); ctx->literalsPtr += length; op += length; @@ -218,7 +214,7 @@ FORCE_INLINE int LIZ_decompress_LZ5v2( /* Overflow error detected */ _output_error: - LIZ_LOG_DECOMPRESS_LZ5v2("_output_error=%d ctx->flagsPtr=%p blockBase=%p\n", (int) (-(ctx->flagsPtr-blockBase))-1, ctx->flagsPtr, blockBase); - LIZ_LOG_DECOMPRESS_LZ5v2("cpy=%p oend=%p ctx->literalsPtr+length[%d]=%p iend=%p\n", cpy, oend, (int)length, ctx->literalsPtr+length, iend); + LIZARD_LOG_DECOMPRESS_LIZv1("_output_error=%d ctx->flagsPtr=%p blockBase=%p\n", (int) (-(ctx->flagsPtr-blockBase))-1, ctx->flagsPtr, blockBase); + LIZARD_LOG_DECOMPRESS_LIZv1("cpy=%p oend=%p ctx->literalsPtr+length[%d]=%p iend=%p\n", cpy, oend, (int)length, ctx->literalsPtr+length, iend); return (int) (-(ctx->flagsPtr-blockBase))-1; } diff --git a/C/lizard/liz_decompress_lz4.h b/C/lizard/lizard_decompress_lz4.h similarity index 72% rename from C/lizard/liz_decompress_lz4.h rename to C/lizard/lizard_decompress_lz4.h index 1ee9499e..74ad2966 100644 --- a/C/lizard/liz_decompress_lz4.h +++ b/C/lizard/lizard_decompress_lz4.h @@ -1,11 +1,11 @@ -/*! LIZ_decompress_LZ4() : +/*! Lizard_decompress_LZ4() : * This generic decompression function cover all use cases. * It shall be instantiated several times, using different sets of directives * Note that it is important this generic function is really inlined, * in order to remove useless branches during compilation optimization. */ -FORCE_INLINE int LIZ_decompress_LZ4( - LIZ_dstream_t* ctx, +FORCE_INLINE int Lizard_decompress_LZ4( + Lizard_dstream_t* ctx, BYTE* const dest, int outputSize, /* this value is the max size of Output Buffer. */ @@ -29,11 +29,10 @@ FORCE_INLINE int LIZ_decompress_LZ4( const BYTE* const lowLimit = lowPrefix - dictSize; const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; - const int checkOffset = (dictSize < (int)(LIZ_DICT_SIZE)); + const int checkOffset = (dictSize < (int)(LIZARD_DICT_SIZE)); intptr_t length = 0; (void)compressionLevel; - (void)LIZ_wildCopy; /* Special cases */ if (unlikely(outputSize==0)) return ((inputSize==1) && (*ctx->flagsPtr==0)) ? 0 : -1; /* Empty output buffer */ @@ -47,7 +46,7 @@ FORCE_INLINE int LIZ_decompress_LZ4( /* get literal length */ token = *ctx->flagsPtr++; if ((length=(token & RUN_MASK_LZ4)) == RUN_MASK_LZ4) { - if (unlikely(ctx->literalsPtr > iend - 5)) { LIZ_LOG_DECOMPRESS_LZ4("0"); goto _output_error; } + if (unlikely(ctx->literalsPtr > iend - 5)) { LIZARD_LOG_DECOMPRESS_LZ4("0"); goto _output_error; } length = *ctx->literalsPtr; if unlikely(length >= 254) { if (length == 254) { @@ -60,23 +59,23 @@ FORCE_INLINE int LIZ_decompress_LZ4( } length += RUN_MASK_LZ4; ctx->literalsPtr++; - if (unlikely((size_t)(op+length)<(size_t)(op))) { LIZ_LOG_DECOMPRESS_LZ4("1"); goto _output_error; } /* overflow detection */ - if (unlikely((size_t)(ctx->literalsPtr+length)<(size_t)(ctx->literalsPtr))) { LIZ_LOG_DECOMPRESS_LZ4("2"); goto _output_error; } /* overflow detection */ + if (unlikely((size_t)(op+length)<(size_t)(op))) { LIZARD_LOG_DECOMPRESS_LZ4("1"); goto _output_error; } /* overflow detection */ + if (unlikely((size_t)(ctx->literalsPtr+length)<(size_t)(ctx->literalsPtr))) { LIZARD_LOG_DECOMPRESS_LZ4("2"); goto _output_error; } /* overflow detection */ } /* copy literals */ cpy = op + length; - if (unlikely(cpy > oend - WILDCOPYLENGTH || ctx->literalsPtr + length > iend - (2 + WILDCOPYLENGTH))) { LIZ_LOG_DECOMPRESS_LZ4("offset outside buffers\n"); goto _output_error; } /* Error : offset outside buffers */ + if (unlikely(cpy > oend - WILDCOPYLENGTH || ctx->literalsPtr + length > iend - (2 + WILDCOPYLENGTH))) { LIZARD_LOG_DECOMPRESS_LZ4("offset outside buffers\n"); goto _output_error; } /* Error : offset outside buffers */ #if 1 - LIZ_wildCopy16(op, ctx->literalsPtr, cpy); + Lizard_wildCopy16(op, ctx->literalsPtr, cpy); op = cpy; ctx->literalsPtr += length; #else - LIZ_copy8(op, ctx->literalsPtr); - LIZ_copy8(op+8, ctx->literalsPtr+8); + Lizard_copy8(op, ctx->literalsPtr); + Lizard_copy8(op+8, ctx->literalsPtr+8); if (length > 16) - LIZ_wildCopy16(op + 16, ctx->literalsPtr + 16, cpy); + Lizard_wildCopy16(op + 16, ctx->literalsPtr + 16, cpy); op = cpy; ctx->literalsPtr += length; #endif @@ -87,12 +86,12 @@ FORCE_INLINE int LIZ_decompress_LZ4( ctx->literalsPtr += 2; match = op - offset; - if ((checkOffset) && (unlikely(match < lowLimit))) { LIZ_LOG_DECOMPRESS_LZ4("lowPrefix[%p]-dictSize[%d]=lowLimit[%p] match[%p]=op[%p]-offset[%d]\n", lowPrefix, (int)dictSize, lowLimit, match, op, (int)offset); goto _output_error; } /* Error : offset outside buffers */ + if ((checkOffset) && (unlikely(match < lowLimit))) { LIZARD_LOG_DECOMPRESS_LZ4("lowPrefix[%p]-dictSize[%d]=lowLimit[%p] match[%p]=op[%p]-offset[%d]\n", lowPrefix, (int)dictSize, lowLimit, match, op, (int)offset); goto _output_error; } /* Error : offset outside buffers */ /* get matchlength */ length = token >> RUN_BITS_LZ4; if (length == ML_MASK_LZ4) { - if (unlikely(ctx->literalsPtr > iend - 5)) { LIZ_LOG_DECOMPRESS_LZ4("4"); goto _output_error; } + if (unlikely(ctx->literalsPtr > iend - 5)) { LIZARD_LOG_DECOMPRESS_LZ4("4"); goto _output_error; } length = *ctx->literalsPtr; if unlikely(length >= 254) { if (length == 254) { @@ -105,13 +104,13 @@ FORCE_INLINE int LIZ_decompress_LZ4( } length += ML_MASK_LZ4; ctx->literalsPtr++; - if (unlikely((size_t)(op+length)<(size_t)(op))) { LIZ_LOG_DECOMPRESS_LZ4("5"); goto _output_error; } /* overflow detection */ + if (unlikely((size_t)(op+length)<(size_t)(op))) { LIZARD_LOG_DECOMPRESS_LZ4("5"); goto _output_error; } /* overflow detection */ } length += MINMATCH; /* check external dictionary */ if ((dict==usingExtDict) && (match < lowPrefix)) { - if (unlikely(op + length > oend - WILDCOPYLENGTH)) { LIZ_LOG_DECOMPRESS_LZ4("6"); goto _output_error; } /* doesn't respect parsing restriction */ + if (unlikely(op + length > oend - WILDCOPYLENGTH)) { LIZARD_LOG_DECOMPRESS_LZ4("6"); goto _output_error; } /* doesn't respect parsing restriction */ if (length <= (intptr_t)(lowPrefix - match)) { /* match can be copied as a single segment from external dictionary */ @@ -136,11 +135,11 @@ FORCE_INLINE int LIZ_decompress_LZ4( /* copy match within block */ cpy = op + length; - if (unlikely(cpy > oend - WILDCOPYLENGTH)) { LIZ_LOG_DECOMPRESS_LZ4("1match=%p lowLimit=%p\n", match, lowLimit); goto _output_error; } /* Error : offset outside buffers */ - LIZ_copy8(op, match); - LIZ_copy8(op+8, match+8); + if (unlikely(cpy > oend - WILDCOPYLENGTH)) { LIZARD_LOG_DECOMPRESS_LZ4("1match=%p lowLimit=%p\n", match, lowLimit); goto _output_error; } /* Error : offset outside buffers */ + Lizard_copy8(op, match); + Lizard_copy8(op+8, match+8); if (length > 16) - LIZ_wildCopy16(op + 16, match + 16, cpy); + Lizard_wildCopy16(op + 16, match + 16, cpy); op = cpy; if ((partialDecoding) && (op >= oexit)) return (int) (op-dest); } @@ -148,7 +147,7 @@ FORCE_INLINE int LIZ_decompress_LZ4( /* last literals */ length = ctx->literalsEnd - ctx->literalsPtr; cpy = op + length; - if ((ctx->literalsPtr+length != iend) || (cpy > oend)) { LIZ_LOG_DECOMPRESS_LZ4("9"); goto _output_error; } /* Error : input must be consumed */ + if ((ctx->literalsPtr+length != iend) || (cpy > oend)) { LIZARD_LOG_DECOMPRESS_LZ4("9"); goto _output_error; } /* Error : input must be consumed */ memcpy(op, ctx->literalsPtr, length); ctx->literalsPtr += length; op += length; @@ -158,7 +157,7 @@ FORCE_INLINE int LIZ_decompress_LZ4( /* Overflow error detected */ _output_error: - LIZ_LOG_DECOMPRESS_LZ4("_output_error=%d ctx->flagsPtr=%p blockBase=%p\n", (int) (-(ctx->flagsPtr-blockBase))-1, ctx->flagsPtr, blockBase); - LIZ_LOG_DECOMPRESS_LZ4("cpy=%p oend=%p ctx->literalsPtr+length[%d]=%p iend=%p\n", cpy, oend, (int)length, ctx->literalsPtr+length, iend); + LIZARD_LOG_DECOMPRESS_LZ4("_output_error=%d ctx->flagsPtr=%p blockBase=%p\n", (int) (-(ctx->flagsPtr-blockBase))-1, ctx->flagsPtr, blockBase); + LIZARD_LOG_DECOMPRESS_LZ4("cpy=%p oend=%p ctx->literalsPtr+length[%d]=%p iend=%p\n", cpy, oend, (int)length, ctx->literalsPtr+length, iend); return (int) (-(ctx->flagsPtr-blockBase))-1; } diff --git a/C/lizard/liz_lz5frame.c b/C/lizard/lizard_frame.c similarity index 64% rename from C/lizard/liz_lz5frame.c rename to C/lizard/lizard_frame.c index b4a9a31d..54f29e8e 100644 --- a/C/lizard/liz_lz5frame.c +++ b/C/lizard/lizard_frame.c @@ -1,6 +1,7 @@ /* -LZ5 auto-framing library -Copyright (C) 2011-2016, Yann Collet. +Lizard auto-framing library +Copyright (C) 2011-2016, Yann Collet +Copyright (C) 2016-2017, Przemyslaw Skibinski BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) @@ -28,10 +29,10 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : -- LZ5 source repository : https://github.com/inikep/lz5 +- Lizard source repository : https://github.com/inikep/lizard */ -/* LZ5F is a stand-alone API to create LZ5-compressed Frames +/* LizardF is a stand-alone API to create Lizard-compressed Frames * in full conformance with specification v1.5.0 * All related operations, including memory management, are handled by the library. * */ @@ -49,10 +50,10 @@ You can contact the author at : /*-************************************ * Includes **************************************/ -#include "lizframe_static.h" -#include "liz_compress.h" -#include "liz_decompress.h" -#include "liz_common.h" /* LIZ_DICT_SIZE */ +#include "lizard_frame_static.h" +#include "lizard_compress.h" +#include "lizard_decompress.h" +#include "lizard_common.h" /* LIZARD_DICT_SIZE */ #define XXH_STATIC_LINKING_ONLY #include "../zstd/xxhash.h" #include @@ -60,7 +61,7 @@ You can contact the author at : /* unoptimized version; solves endianess & alignment issues */ -static U32 LIZF_readLE32 (const void* src) +static U32 LizardF_readLE32 (const void* src) { const BYTE* const srcPtr = (const BYTE*)src; U32 value32 = srcPtr[0]; @@ -70,7 +71,7 @@ static U32 LIZF_readLE32 (const void* src) return value32; } -static void LIZF_writeLE32 (BYTE* dstPtr, U32 value32) +static void LizardF_writeLE32 (BYTE* dstPtr, U32 value32) { dstPtr[0] = (BYTE)value32; dstPtr[1] = (BYTE)(value32 >> 8); @@ -78,7 +79,7 @@ static void LIZF_writeLE32 (BYTE* dstPtr, U32 value32) dstPtr[3] = (BYTE)(value32 >> 24); } -static U64 LIZF_readLE64 (const BYTE* srcPtr) +static U64 LizardF_readLE64 (const BYTE* srcPtr) { U64 value64 = srcPtr[0]; value64 += ((U64)srcPtr[1]<<8); @@ -91,7 +92,7 @@ static U64 LIZF_readLE64 (const BYTE* srcPtr) return value64; } -static void LIZF_writeLE64 (BYTE* dstPtr, U64 value64) +static void LizardF_writeLE64 (BYTE* dstPtr, U64 value64) { dstPtr[0] = (BYTE)value64; dstPtr[1] = (BYTE)(value64 >> 8); @@ -113,10 +114,10 @@ static void LIZF_writeLE64 (BYTE* dstPtr, U64 value64) #define _4BITS 0x0F #define _8BITS 0xFF -#define LIZF_MAGIC_SKIPPABLE_START 0x184D2A50U -#define LIZF_MAGICNUMBER 0x184D2206U -#define LIZF_BLOCKUNCOMPRESSED_FLAG 0x80000000U -#define LIZF_BLOCKSIZEID_DEFAULT LIZF_max128KB +#define LIZARDF_MAGIC_SKIPPABLE_START 0x184D2A50U +#define LIZARDF_MAGICNUMBER 0x184D2206U +#define LIZARDF_BLOCKUNCOMPRESSED_FLAG 0x80000000U +#define LIZARDF_BLOCKSIZEID_DEFAULT LizardF_max128KB static const size_t minFHSize = 7; static const size_t maxFHSize = 15; @@ -126,9 +127,9 @@ static const size_t BHSize = 4; /*-************************************ * Structures and local types **************************************/ -typedef struct LIZF_cctx_s +typedef struct LizardF_cctx_s { - LIZF_preferences_t prefs; + LizardF_preferences_t prefs; U32 version; U32 cStage; size_t maxBlockSize; @@ -138,13 +139,13 @@ typedef struct LIZF_cctx_s size_t tmpInSize; U64 totalInSize; XXH32_state_t xxh; - LIZ_stream_t* lz5CtxPtr; - U32 lz5CtxLevel; /* 0: unallocated; 1: LIZ_stream_t; */ -} LIZF_cctx_t; + Lizard_stream_t* lizardCtxPtr; + U32 lizardCtxLevel; /* 0: unallocated; 1: Lizard_stream_t; */ +} LizardF_cctx_t; -typedef struct LIZF_dctx_s +typedef struct LizardF_dctx_s { - LIZF_frameInfo_t frameInfo; + LizardF_frameInfo_t frameInfo; U32 version; U32 dStage; U64 frameRemainingSize; @@ -162,25 +163,25 @@ typedef struct LIZF_dctx_s size_t tmpOutStart; XXH32_state_t xxh; BYTE header[16]; -} LIZF_dctx_t; +} LizardF_dctx_t; /*-************************************ * Error management **************************************/ -#define LIZF_GENERATE_STRING(STRING) #STRING, -static const char* LIZF_errorStrings[] = { LIZF_LIST_ERRORS(LIZF_GENERATE_STRING) }; +#define LIZARDF_GENERATE_STRING(STRING) #STRING, +static const char* LizardF_errorStrings[] = { LIZARDF_LIST_ERRORS(LIZARDF_GENERATE_STRING) }; -unsigned LIZF_isError(LIZF_errorCode_t code) +unsigned LizardF_isError(LizardF_errorCode_t code) { - return (code > (LIZF_errorCode_t)(-LIZF_ERROR_maxCode)); + return (code > (LizardF_errorCode_t)(-LizardF_ERROR_maxCode)); } -const char* LIZF_getErrorName(LIZF_errorCode_t code) +const char* LizardF_getErrorName(LizardF_errorCode_t code) { static const char* codeError = "Unspecified error code"; - if (LIZF_isError(code)) return LIZF_errorStrings[-(int)(code)]; + if (LizardF_isError(code)) return LizardF_errorStrings[-(int)(code)]; return codeError; } @@ -188,35 +189,35 @@ const char* LIZF_getErrorName(LIZF_errorCode_t code) /*-************************************ * Private functions **************************************/ -static size_t LIZF_getBlockSize(unsigned blockSizeID) +static size_t LizardF_getBlockSize(unsigned blockSizeID) { static const size_t blockSizes[7] = { 128 KB, 256 KB, 1 MB, 4 MB, 16 MB, 64 MB, 256 MB }; - if (blockSizeID == 0) blockSizeID = LIZF_BLOCKSIZEID_DEFAULT; + if (blockSizeID == 0) blockSizeID = LIZARDF_BLOCKSIZEID_DEFAULT; blockSizeID -= 1; - if (blockSizeID >= 7) return (size_t)-LIZF_ERROR_maxBlockSize_invalid; + if (blockSizeID >= 7) return (size_t)-LizardF_ERROR_maxBlockSize_invalid; return blockSizes[blockSizeID]; } -static LIZF_blockSizeID_t LIZF_optimalBSID(const LIZF_blockSizeID_t requestedBSID, const size_t srcSize) +static LizardF_blockSizeID_t LizardF_optimalBSID(const LizardF_blockSizeID_t requestedBSID, const size_t srcSize) { - LIZF_blockSizeID_t proposedBSID = LIZF_max128KB; + LizardF_blockSizeID_t proposedBSID = LizardF_max128KB; size_t maxBlockSize; while (requestedBSID > proposedBSID) { - maxBlockSize = LIZF_getBlockSize(proposedBSID); + maxBlockSize = LizardF_getBlockSize(proposedBSID); if (srcSize <= maxBlockSize) { return proposedBSID; } - proposedBSID = (LIZF_blockSizeID_t)((int)proposedBSID + 1); + proposedBSID = (LizardF_blockSizeID_t)((int)proposedBSID + 1); } return requestedBSID; } -static BYTE LIZF_headerChecksum (const void* header, size_t length) +static BYTE LizardF_headerChecksum (const void* header, size_t length) { U32 xxh = XXH32(header, length, 0); return (BYTE)(xxh >> 8); @@ -227,41 +228,41 @@ static BYTE LIZF_headerChecksum (const void* header, size_t length) * Simple compression functions **************************************/ -size_t LIZF_compressFrameBound(size_t srcSize, const LIZF_preferences_t* preferencesPtr) +size_t LizardF_compressFrameBound(size_t srcSize, const LizardF_preferences_t* preferencesPtr) { - LIZF_preferences_t prefs; + LizardF_preferences_t prefs; size_t headerSize; size_t streamSize; if (preferencesPtr!=NULL) prefs = *preferencesPtr; else memset(&prefs, 0, sizeof(prefs)); - prefs.frameInfo.blockSizeID = LIZF_optimalBSID(prefs.frameInfo.blockSizeID, srcSize); + prefs.frameInfo.blockSizeID = LizardF_optimalBSID(prefs.frameInfo.blockSizeID, srcSize); prefs.autoFlush = 1; headerSize = maxFHSize; /* header size, including magic number and frame content size*/ - streamSize = LIZF_compressBound(srcSize, &prefs); + streamSize = LizardF_compressBound(srcSize, &prefs); return headerSize + streamSize; } -/*! LIZF_compressFrame() : -* Compress an entire srcBuffer into a valid LZ5 frame, as defined by specification v1.5.0, in a single step. +/*! LizardF_compressFrame() : +* Compress an entire srcBuffer into a valid Lizard frame, as defined by specification v1.5.0, in a single step. * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. -* You can get the minimum value of dstMaxSize by using LIZF_compressFrameBound() -* If this condition is not respected, LIZF_compressFrame() will fail (result is an errorCode) -* The LIZF_preferences_t structure is optional : you can provide NULL as argument. All preferences will then be set to default. +* You can get the minimum value of dstMaxSize by using LizardF_compressFrameBound() +* If this condition is not respected, LizardF_compressFrame() will fail (result is an errorCode) +* The LizardF_preferences_t structure is optional : you can provide NULL as argument. All preferences will then be set to default. * The result of the function is the number of bytes written into dstBuffer. -* The function outputs an error code if it fails (can be tested using LIZF_isError()) +* The function outputs an error code if it fails (can be tested using LizardF_isError()) */ -size_t LIZF_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LIZF_preferences_t* preferencesPtr) +size_t LizardF_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LizardF_preferences_t* preferencesPtr) { - LIZF_cctx_t cctxI; - LIZF_preferences_t prefs; - LIZF_compressOptions_t options; - LIZF_errorCode_t errorCode; + LizardF_cctx_t cctxI; + LizardF_preferences_t prefs; + LizardF_compressOptions_t options; + LizardF_errorCode_t errorCode; BYTE* const dstStart = (BYTE*) dstBuffer; BYTE* dstPtr = dstStart; BYTE* const dstEnd = dstStart + dstMaxSize; @@ -269,7 +270,7 @@ size_t LIZF_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf memset(&cctxI, 0, sizeof(cctxI)); /* works because no allocation */ memset(&options, 0, sizeof(options)); - cctxI.version = LIZF_VERSION; + cctxI.version = LIZARDF_VERSION; cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent allocation; works because autoflush==1 & stableSrc==1 */ if (preferencesPtr!=NULL) @@ -279,33 +280,33 @@ size_t LIZF_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf if (prefs.frameInfo.contentSize != 0) prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */ - prefs.frameInfo.blockSizeID = LIZF_optimalBSID(prefs.frameInfo.blockSizeID, srcSize); + prefs.frameInfo.blockSizeID = LizardF_optimalBSID(prefs.frameInfo.blockSizeID, srcSize); prefs.autoFlush = 1; - if (srcSize <= LIZF_getBlockSize(prefs.frameInfo.blockSizeID)) - prefs.frameInfo.blockMode = LIZF_blockIndependent; /* no need for linked blocks */ + if (srcSize <= LizardF_getBlockSize(prefs.frameInfo.blockSizeID)) + prefs.frameInfo.blockMode = LizardF_blockIndependent; /* no need for linked blocks */ options.stableSrc = 1; - if (dstMaxSize < LIZF_compressFrameBound(srcSize, &prefs)) - return (size_t)-LIZF_ERROR_dstMaxSize_tooSmall; + if (dstMaxSize < LizardF_compressFrameBound(srcSize, &prefs)) + return (size_t)-LizardF_ERROR_dstMaxSize_tooSmall; - errorCode = LIZF_compressBegin(&cctxI, dstBuffer, dstMaxSize, &prefs); /* write header */ - if (LIZF_isError(errorCode)) goto error; + errorCode = LizardF_compressBegin(&cctxI, dstBuffer, dstMaxSize, &prefs); /* write header */ + if (LizardF_isError(errorCode)) goto error; dstPtr += errorCode; /* header size */ - errorCode = LIZF_compressUpdate(&cctxI, dstPtr, dstEnd-dstPtr, srcBuffer, srcSize, &options); - if (LIZF_isError(errorCode)) goto error; + errorCode = LizardF_compressUpdate(&cctxI, dstPtr, dstEnd-dstPtr, srcBuffer, srcSize, &options); + if (LizardF_isError(errorCode)) goto error; dstPtr += errorCode; - errorCode = LIZF_compressEnd(&cctxI, dstPtr, dstEnd-dstPtr, &options); /* flush last block, and generate suffix */ - if (LIZF_isError(errorCode)) goto error; + errorCode = LizardF_compressEnd(&cctxI, dstPtr, dstEnd-dstPtr, &options); /* flush last block, and generate suffix */ + if (LizardF_isError(errorCode)) goto error; dstPtr += errorCode; - LIZ_freeStream(cctxI.lz5CtxPtr); + Lizard_freeStream(cctxI.lizardCtxPtr); FREEMEM(cctxI.tmpBuff); return (dstPtr - dstStart); error: - LIZ_freeStream(cctxI.lz5CtxPtr); + Lizard_freeStream(cctxI.lizardCtxPtr); FREEMEM(cctxI.tmpBuff); return errorCode; } @@ -315,94 +316,93 @@ error: * Advanced compression functions ***********************************/ -/* LIZF_createCompressionContext() : +/* LizardF_createCompressionContext() : * The first thing to do is to create a compressionContext object, which will be used in all compression operations. -* This is achieved using LIZF_createCompressionContext(), which takes as argument a version and an LIZF_preferences_t structure. -* The version provided MUST be LIZF_VERSION. It is intended to track potential version differences between different binaries. -* The function will provide a pointer to an allocated LIZF_compressionContext_t object. -* If the result LIZF_errorCode_t is not OK_NoError, there was an error during context creation. -* Object can release its memory using LIZF_freeCompressionContext(); +* This is achieved using LizardF_createCompressionContext(), which takes as argument a version and an LizardF_preferences_t structure. +* The version provided MUST be LIZARDF_VERSION. It is intended to track potential version differences between different binaries. +* The function will provide a pointer to an allocated LizardF_compressionContext_t object. +* If the result LizardF_errorCode_t is not OK_NoError, there was an error during context creation. +* Object can release its memory using LizardF_freeCompressionContext(); */ -LIZF_errorCode_t LIZF_createCompressionContext(LIZF_compressionContext_t* LIZF_compressionContextPtr, unsigned version) +LizardF_errorCode_t LizardF_createCompressionContext(LizardF_compressionContext_t* LizardF_compressionContextPtr, unsigned version) { - LIZF_cctx_t* cctxPtr; + LizardF_cctx_t* cctxPtr; - cctxPtr = (LIZF_cctx_t*)ALLOCATOR(1, sizeof(LIZF_cctx_t)); - if (cctxPtr==NULL) return (LIZF_errorCode_t)(-LIZF_ERROR_allocation_failed); + cctxPtr = (LizardF_cctx_t*)ALLOCATOR(1, sizeof(LizardF_cctx_t)); + if (cctxPtr==NULL) return (LizardF_errorCode_t)(-LizardF_ERROR_allocation_failed); cctxPtr->version = version; cctxPtr->cStage = 0; /* Next stage : write header */ - *LIZF_compressionContextPtr = (LIZF_compressionContext_t)cctxPtr; + *LizardF_compressionContextPtr = (LizardF_compressionContext_t)cctxPtr; - return LIZF_OK_NoError; + return LizardF_OK_NoError; } -LIZF_errorCode_t LIZF_freeCompressionContext(LIZF_compressionContext_t LIZF_compressionContext) +LizardF_errorCode_t LizardF_freeCompressionContext(LizardF_compressionContext_t LizardF_compressionContext) { - LIZF_cctx_t* cctxPtr = (LIZF_cctx_t*)LIZF_compressionContext; + LizardF_cctx_t* cctxPtr = (LizardF_cctx_t*)LizardF_compressionContext; if (cctxPtr != NULL) { /* null pointers can be safely provided to this function, like free() */ - LIZ_freeStream(cctxPtr->lz5CtxPtr); + Lizard_freeStream(cctxPtr->lizardCtxPtr); FREEMEM(cctxPtr->tmpBuff); - FREEMEM(LIZF_compressionContext); + FREEMEM(LizardF_compressionContext); } - return LIZF_OK_NoError; + return LizardF_OK_NoError; } -/*! LIZF_compressBegin() : +/*! LizardF_compressBegin() : * will write the frame header into dstBuffer. -* dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is LIZF_MAXHEADERFRAME_SIZE bytes. +* dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is LizardF_MAXHEADERFRAME_SIZE bytes. * The result of the function is the number of bytes written into dstBuffer for the header -* or an error code (can be tested using LIZF_isError()) +* or an error code (can be tested using LizardF_isError()) */ -size_t LIZF_compressBegin(LIZF_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LIZF_preferences_t* preferencesPtr) +size_t LizardF_compressBegin(LizardF_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LizardF_preferences_t* preferencesPtr) { - LIZF_preferences_t prefNull; - LIZF_cctx_t* cctxPtr = (LIZF_cctx_t*)compressionContext; + LizardF_preferences_t prefNull; + LizardF_cctx_t* cctxPtr = (LizardF_cctx_t*)compressionContext; BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; BYTE* headerStart; size_t requiredBuffSize; - if (dstMaxSize < maxFHSize) return (size_t)-LIZF_ERROR_dstMaxSize_tooSmall; - if (cctxPtr->cStage != 0) return (size_t)-LIZF_ERROR_GENERIC; + if (dstMaxSize < maxFHSize) return (size_t)-LizardF_ERROR_dstMaxSize_tooSmall; + if (cctxPtr->cStage != 0) return (size_t)-LizardF_ERROR_GENERIC; memset(&prefNull, 0, sizeof(prefNull)); if (preferencesPtr == NULL) preferencesPtr = &prefNull; cctxPtr->prefs = *preferencesPtr; /* ctx Management */ - if (cctxPtr->lz5CtxLevel == 0) { - cctxPtr->lz5CtxPtr = LIZ_createStream(cctxPtr->prefs.compressionLevel); - cctxPtr->lz5CtxLevel = 1; + if (cctxPtr->lizardCtxLevel == 0) { + cctxPtr->lizardCtxPtr = Lizard_createStream(cctxPtr->prefs.compressionLevel); + cctxPtr->lizardCtxLevel = 1; } /* Buffer Management */ - if (cctxPtr->prefs.frameInfo.blockSizeID == 0) cctxPtr->prefs.frameInfo.blockSizeID = LIZF_BLOCKSIZEID_DEFAULT; - cctxPtr->maxBlockSize = LIZF_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID); - requiredBuffSize = cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LIZF_blockLinked) * 2 * LIZ_DICT_SIZE); - // printf("LIZF_compressBegin maxBlockSize=%d requiredBuffSize=%d LIZF_blockLinked=%d\n", (int)cctxPtr->maxBlockSize, (int)requiredBuffSize, (cctxPtr->prefs.frameInfo.blockMode == LIZF_blockLinked)); + if (cctxPtr->prefs.frameInfo.blockSizeID == 0) cctxPtr->prefs.frameInfo.blockSizeID = LIZARDF_BLOCKSIZEID_DEFAULT; + cctxPtr->maxBlockSize = LizardF_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID); + requiredBuffSize = cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LizardF_blockLinked) * 2 * LIZARD_DICT_SIZE); if (preferencesPtr->autoFlush) - requiredBuffSize = (cctxPtr->prefs.frameInfo.blockMode == LIZF_blockLinked) * LIZ_DICT_SIZE; /* just needs dict */ + requiredBuffSize = (cctxPtr->prefs.frameInfo.blockMode == LizardF_blockLinked) * LIZARD_DICT_SIZE; /* just needs dict */ if (cctxPtr->maxBufferSize < requiredBuffSize) { cctxPtr->maxBufferSize = requiredBuffSize; FREEMEM(cctxPtr->tmpBuff); cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(1, requiredBuffSize); - if (cctxPtr->tmpBuff == NULL) { printf("ERROR in LIZF_compressBegin: Cannot allocate %d MB\n", (int)(requiredBuffSize>>20)); return (size_t)-LIZF_ERROR_allocation_failed; } + if (cctxPtr->tmpBuff == NULL) { printf("ERROR in LizardF_compressBegin: Cannot allocate %d MB\n", (int)(requiredBuffSize>>20)); return (size_t)-LizardF_ERROR_allocation_failed; } } cctxPtr->tmpIn = cctxPtr->tmpBuff; cctxPtr->tmpInSize = 0; XXH32_reset(&(cctxPtr->xxh), 0); - cctxPtr->lz5CtxPtr = LIZ_resetStream((LIZ_stream_t*)(cctxPtr->lz5CtxPtr), cctxPtr->prefs.compressionLevel); - if (!cctxPtr->lz5CtxPtr) return (size_t)-LIZF_ERROR_allocation_failed; + cctxPtr->lizardCtxPtr = Lizard_resetStream((Lizard_stream_t*)(cctxPtr->lizardCtxPtr), cctxPtr->prefs.compressionLevel); + if (!cctxPtr->lizardCtxPtr) return (size_t)-LizardF_ERROR_allocation_failed; /* Magic Number */ - LIZF_writeLE32(dstPtr, LIZF_MAGICNUMBER); + LizardF_writeLE32(dstPtr, LIZARDF_MAGICNUMBER); dstPtr += 4; headerStart = dstPtr; @@ -415,12 +415,12 @@ size_t LIZF_compressBegin(LIZF_compressionContext_t compressionContext, void* ds *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4); /* Optional Frame content size field */ if (cctxPtr->prefs.frameInfo.contentSize) { - LIZF_writeLE64(dstPtr, cctxPtr->prefs.frameInfo.contentSize); + LizardF_writeLE64(dstPtr, cctxPtr->prefs.frameInfo.contentSize); dstPtr += 8; cctxPtr->totalInSize = 0; } /* CRC Byte */ - *dstPtr = LIZF_headerChecksum(headerStart, dstPtr - headerStart); + *dstPtr = LizardF_headerChecksum(headerStart, dstPtr - headerStart); dstPtr++; cctxPtr->cStage = 1; /* header written, now request input data block */ @@ -429,18 +429,18 @@ size_t LIZF_compressBegin(LIZF_compressionContext_t compressionContext, void* ds } -/* LIZF_compressBound() : gives the size of Dst buffer given a srcSize to handle worst case situations. -* The LIZF_frameInfo_t structure is optional : +/* LizardF_compressBound() : gives the size of Dst buffer given a srcSize to handle worst case situations. +* The LizardF_frameInfo_t structure is optional : * you can provide NULL as argument, preferences will then be set to cover worst case situations. * */ -size_t LIZF_compressBound(size_t srcSize, const LIZF_preferences_t* preferencesPtr) +size_t LizardF_compressBound(size_t srcSize, const LizardF_preferences_t* preferencesPtr) { - LIZF_preferences_t prefsNull; + LizardF_preferences_t prefsNull; memset(&prefsNull, 0, sizeof(prefsNull)); - prefsNull.frameInfo.contentChecksumFlag = LIZF_contentChecksumEnabled; /* worst case */ - { const LIZF_preferences_t* prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; - LIZF_blockSizeID_t bid = prefsPtr->frameInfo.blockSizeID; - size_t blockSize = LIZF_getBlockSize(bid); + prefsNull.frameInfo.contentChecksumFlag = LizardF_contentChecksumEnabled; /* worst case */ + { const LizardF_preferences_t* prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; + LizardF_blockSizeID_t bid = prefsPtr->frameInfo.blockSizeID; + size_t blockSize = LizardF_getBlockSize(bid); unsigned nbBlocks = (unsigned)(srcSize / blockSize) + 1; size_t lastBlockSize = prefsPtr->autoFlush ? srcSize % blockSize : blockSize; size_t blockInfo = 4; /* default, without block CRC option */ @@ -453,16 +453,16 @@ size_t LIZF_compressBound(size_t srcSize, const LIZF_preferences_t* preferencesP typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level); -static size_t LIZF_compressBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz5ctx, int level) +static size_t LizardF_compressBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lizardctx, int level) { /* compress one block */ BYTE* cSizePtr = (BYTE*)dst; U32 cSize; - cSize = (U32)compress(lz5ctx, (const char*)src, (char*)(cSizePtr+4), (int)(srcSize), (int)(srcSize-1), level); - LIZF_writeLE32(cSizePtr, cSize); + cSize = (U32)compress(lizardctx, (const char*)src, (char*)(cSizePtr+4), (int)(srcSize), (int)(srcSize-1), level); + LizardF_writeLE32(cSizePtr, cSize); if (cSize == 0) { /* compression failed */ cSize = (U32)srcSize; - LIZF_writeLE32(cSizePtr, cSize + LIZF_BLOCKUNCOMPRESSED_FLAG); + LizardF_writeLE32(cSizePtr, cSize + LIZARDF_BLOCKUNCOMPRESSED_FLAG); memcpy(cSizePtr+4, src, srcSize); } return cSize + 4; @@ -470,54 +470,54 @@ static size_t LIZF_compressBlock(void* dst, const void* src, size_t srcSize, com -static int LIZF_localLIZ_compress_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) +static int LizardF_localLizard_compress_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) { (void)level; - return LIZ_compress_continue((LIZ_stream_t*)ctx, src, dst, srcSize, dstSize); + return Lizard_compress_continue((Lizard_stream_t*)ctx, src, dst, srcSize, dstSize); } -static compressFunc_t LIZF_selectCompression(LIZF_blockMode_t blockMode) +static compressFunc_t LizardF_selectCompression(LizardF_blockMode_t blockMode) { - if (blockMode == LIZF_blockIndependent) return LIZ_compress_extState; - return LIZF_localLIZ_compress_continue; + if (blockMode == LizardF_blockIndependent) return Lizard_compress_extState; + return LizardF_localLizard_compress_continue; } -static int LIZF_localSaveDict(LIZF_cctx_t* cctxPtr) +static int LizardF_localSaveDict(LizardF_cctx_t* cctxPtr) { - return LIZ_saveDict ((LIZ_stream_t*)(cctxPtr->lz5CtxPtr), (char*)(cctxPtr->tmpBuff), LIZ_DICT_SIZE); + return Lizard_saveDict ((Lizard_stream_t*)(cctxPtr->lizardCtxPtr), (char*)(cctxPtr->tmpBuff), LIZARD_DICT_SIZE); } -typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LIZF_lastBlockStatus; +typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LizardF_lastBlockStatus; -/*! LIZF_compressUpdate() : -* LIZF_compressUpdate() can be called repetitively to compress as much data as necessary. +/*! LizardF_compressUpdate() : +* LizardF_compressUpdate() can be called repetitively to compress as much data as necessary. * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. -* If this condition is not respected, LIZF_compress() will fail (result is an errorCode) -* You can get the minimum value of dstMaxSize by using LIZF_compressBound() -* The LIZF_compressOptions_t structure is optional : you can provide NULL as argument. +* If this condition is not respected, LizardF_compress() will fail (result is an errorCode) +* You can get the minimum value of dstMaxSize by using LizardF_compressBound() +* The LizardF_compressOptions_t structure is optional : you can provide NULL as argument. * The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered. -* The function outputs an error code if it fails (can be tested using LIZF_isError()) +* The function outputs an error code if it fails (can be tested using LizardF_isError()) */ -size_t LIZF_compressUpdate(LIZF_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LIZF_compressOptions_t* compressOptionsPtr) +size_t LizardF_compressUpdate(LizardF_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LizardF_compressOptions_t* compressOptionsPtr) { - LIZF_compressOptions_t cOptionsNull; - LIZF_cctx_t* cctxPtr = (LIZF_cctx_t*)compressionContext; + LizardF_compressOptions_t cOptionsNull; + LizardF_cctx_t* cctxPtr = (LizardF_cctx_t*)compressionContext; size_t blockSize = cctxPtr->maxBlockSize; const BYTE* srcPtr = (const BYTE*)srcBuffer; const BYTE* const srcEnd = srcPtr + srcSize; BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; - LIZF_lastBlockStatus lastBlockCompressed = notDone; + LizardF_lastBlockStatus lastBlockCompressed = notDone; compressFunc_t compress; - if (cctxPtr->cStage != 1) return (size_t)-LIZF_ERROR_GENERIC; - if (dstMaxSize < LIZF_compressBound(srcSize, &(cctxPtr->prefs))) return (size_t)-LIZF_ERROR_dstMaxSize_tooSmall; + if (cctxPtr->cStage != 1) return (size_t)-LizardF_ERROR_GENERIC; + if (dstMaxSize < LizardF_compressBound(srcSize, &(cctxPtr->prefs))) return (size_t)-LizardF_ERROR_dstMaxSize_tooSmall; memset(&cOptionsNull, 0, sizeof(cOptionsNull)); if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; /* select compression function */ - compress = LIZF_selectCompression(cctxPtr->prefs.frameInfo.blockMode); + compress = LizardF_selectCompression(cctxPtr->prefs.frameInfo.blockMode); /* complete tmp buffer */ if (cctxPtr->tmpInSize > 0) { /* some data already within tmp buffer */ @@ -534,9 +534,9 @@ size_t LIZF_compressUpdate(LIZF_compressionContext_t compressionContext, void* d memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy); srcPtr += sizeToCopy; - dstPtr += LIZF_compressBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, cctxPtr->lz5CtxPtr, cctxPtr->prefs.compressionLevel); + dstPtr += LizardF_compressBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, cctxPtr->lizardCtxPtr, cctxPtr->prefs.compressionLevel); - if (cctxPtr->prefs.frameInfo.blockMode==LIZF_blockLinked) cctxPtr->tmpIn += blockSize; + if (cctxPtr->prefs.frameInfo.blockMode==LizardF_blockLinked) cctxPtr->tmpIn += blockSize; cctxPtr->tmpInSize = 0; } } @@ -544,33 +544,33 @@ size_t LIZF_compressUpdate(LIZF_compressionContext_t compressionContext, void* d while ((size_t)(srcEnd - srcPtr) >= blockSize) { /* compress full block */ lastBlockCompressed = fromSrcBuffer; - dstPtr += LIZF_compressBlock(dstPtr, srcPtr, blockSize, compress, cctxPtr->lz5CtxPtr, cctxPtr->prefs.compressionLevel); + dstPtr += LizardF_compressBlock(dstPtr, srcPtr, blockSize, compress, cctxPtr->lizardCtxPtr, cctxPtr->prefs.compressionLevel); srcPtr += blockSize; } if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) { /* compress remaining input < blockSize */ lastBlockCompressed = fromSrcBuffer; - dstPtr += LIZF_compressBlock(dstPtr, srcPtr, srcEnd - srcPtr, compress, cctxPtr->lz5CtxPtr, cctxPtr->prefs.compressionLevel); + dstPtr += LizardF_compressBlock(dstPtr, srcPtr, srcEnd - srcPtr, compress, cctxPtr->lizardCtxPtr, cctxPtr->prefs.compressionLevel); srcPtr = srcEnd; } /* preserve dictionary if necessary */ - if ((cctxPtr->prefs.frameInfo.blockMode==LIZF_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) { + if ((cctxPtr->prefs.frameInfo.blockMode==LizardF_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) { if (compressOptionsPtr->stableSrc) { cctxPtr->tmpIn = cctxPtr->tmpBuff; } else { - int realDictSize = LIZF_localSaveDict(cctxPtr); - if (realDictSize==0) return (size_t)-LIZF_ERROR_GENERIC; + int realDictSize = LizardF_localSaveDict(cctxPtr); + if (realDictSize==0) return (size_t)-LizardF_ERROR_GENERIC; cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; } } /* keep tmpIn within limits */ - if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) /* necessarily LIZF_blockLinked && lastBlockCompressed==fromTmpBuffer */ + if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) /* necessarily LizardF_blockLinked && lastBlockCompressed==fromTmpBuffer */ && !(cctxPtr->prefs.autoFlush)) { - int realDictSize = LIZF_localSaveDict(cctxPtr); + int realDictSize = LizardF_localSaveDict(cctxPtr); cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; } @@ -582,7 +582,7 @@ size_t LIZF_compressUpdate(LIZF_compressionContext_t compressionContext, void* d cctxPtr->tmpInSize = sizeToCopy; } - if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LIZF_contentChecksumEnabled) + if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LizardF_contentChecksumEnabled) XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize); cctxPtr->totalInSize += srcSize; @@ -590,38 +590,38 @@ size_t LIZF_compressUpdate(LIZF_compressionContext_t compressionContext, void* d } -/*! LIZF_flush() : +/*! LizardF_flush() : * Should you need to create compressed data immediately, without waiting for a block to be filled, -* you can call LIZ_flush(), which will immediately compress any remaining data stored within compressionContext. +* you can call Lizard_flush(), which will immediately compress any remaining data stored within compressionContext. * The result of the function is the number of bytes written into dstBuffer * (it can be zero, this means there was no data left within compressionContext) -* The function outputs an error code if it fails (can be tested using LIZF_isError()) -* The LIZF_compressOptions_t structure is optional : you can provide NULL as argument. +* The function outputs an error code if it fails (can be tested using LizardF_isError()) +* The LizardF_compressOptions_t structure is optional : you can provide NULL as argument. */ -size_t LIZF_flush(LIZF_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LIZF_compressOptions_t* compressOptionsPtr) +size_t LizardF_flush(LizardF_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LizardF_compressOptions_t* compressOptionsPtr) { - LIZF_cctx_t* cctxPtr = (LIZF_cctx_t*)compressionContext; + LizardF_cctx_t* cctxPtr = (LizardF_cctx_t*)compressionContext; BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; compressFunc_t compress; if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */ - if (cctxPtr->cStage != 1) return (size_t)-LIZF_ERROR_GENERIC; - if (dstMaxSize < (cctxPtr->tmpInSize + 8)) return (size_t)-LIZF_ERROR_dstMaxSize_tooSmall; /* +8 : block header(4) + block checksum(4) */ + if (cctxPtr->cStage != 1) return (size_t)-LizardF_ERROR_GENERIC; + if (dstMaxSize < (cctxPtr->tmpInSize + 8)) return (size_t)-LizardF_ERROR_dstMaxSize_tooSmall; /* +8 : block header(4) + block checksum(4) */ (void)compressOptionsPtr; /* not yet useful */ /* select compression function */ - compress = LIZF_selectCompression(cctxPtr->prefs.frameInfo.blockMode); + compress = LizardF_selectCompression(cctxPtr->prefs.frameInfo.blockMode); /* compress tmp buffer */ - dstPtr += LIZF_compressBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lz5CtxPtr, cctxPtr->prefs.compressionLevel); - if (cctxPtr->prefs.frameInfo.blockMode==LIZF_blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize; + dstPtr += LizardF_compressBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lizardCtxPtr, cctxPtr->prefs.compressionLevel); + if (cctxPtr->prefs.frameInfo.blockMode==LizardF_blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize; cctxPtr->tmpInSize = 0; /* keep tmpIn within limits */ - if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) { /* necessarily LIZF_blockLinked */ - int realDictSize = LIZF_localSaveDict(cctxPtr); + if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) { /* necessarily LizardF_blockLinked */ + int realDictSize = LizardF_localSaveDict(cctxPtr); cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; } @@ -629,32 +629,32 @@ size_t LIZF_flush(LIZF_compressionContext_t compressionContext, void* dstBuffer, } -/*! LIZF_compressEnd() : -* When you want to properly finish the compressed frame, just call LIZF_compressEnd(). -* It will flush whatever data remained within compressionContext (like LIZ_flush()) +/*! LizardF_compressEnd() : +* When you want to properly finish the compressed frame, just call LizardF_compressEnd(). +* It will flush whatever data remained within compressionContext (like Lizard_flush()) * but also properly finalize the frame, with an endMark and a checksum. * The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size)) -* The function outputs an error code if it fails (can be tested using LIZF_isError()) -* The LIZF_compressOptions_t structure is optional : you can provide NULL as argument. -* compressionContext can then be used again, starting with LIZF_compressBegin(). The preferences will remain the same. +* The function outputs an error code if it fails (can be tested using LizardF_isError()) +* The LizardF_compressOptions_t structure is optional : you can provide NULL as argument. +* compressionContext can then be used again, starting with LizardF_compressBegin(). The preferences will remain the same. */ -size_t LIZF_compressEnd(LIZF_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LIZF_compressOptions_t* compressOptionsPtr) +size_t LizardF_compressEnd(LizardF_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LizardF_compressOptions_t* compressOptionsPtr) { - LIZF_cctx_t* cctxPtr = (LIZF_cctx_t*)compressionContext; + LizardF_cctx_t* cctxPtr = (LizardF_cctx_t*)compressionContext; BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; size_t errorCode; - errorCode = LIZF_flush(compressionContext, dstBuffer, dstMaxSize, compressOptionsPtr); - if (LIZF_isError(errorCode)) return errorCode; + errorCode = LizardF_flush(compressionContext, dstBuffer, dstMaxSize, compressOptionsPtr); + if (LizardF_isError(errorCode)) return errorCode; dstPtr += errorCode; - LIZF_writeLE32(dstPtr, 0); + LizardF_writeLE32(dstPtr, 0); dstPtr+=4; /* endMark */ - if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LIZF_contentChecksumEnabled) { + if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LizardF_contentChecksumEnabled) { U32 xxh = XXH32_digest(&(cctxPtr->xxh)); - LIZF_writeLE32(dstPtr, xxh); + LizardF_writeLE32(dstPtr, xxh); dstPtr+=4; /* content Checksum */ } @@ -663,7 +663,7 @@ size_t LIZF_compressEnd(LIZF_compressionContext_t compressionContext, void* dstB if (cctxPtr->prefs.frameInfo.contentSize) { if (cctxPtr->prefs.frameInfo.contentSize != cctxPtr->totalInSize) - return (size_t)-LIZF_ERROR_frameSize_wrong; + return (size_t)-LizardF_ERROR_frameSize_wrong; } return dstPtr - dstStart; @@ -676,28 +676,28 @@ size_t LIZF_compressEnd(LIZF_compressionContext_t compressionContext, void* dstB /* Resource management */ -/*! LIZF_createDecompressionContext() : +/*! LizardF_createDecompressionContext() : * Create a decompressionContext object, which will track all decompression operations. -* Provides a pointer to a fully allocated and initialized LIZF_decompressionContext object. -* Object can later be released using LIZF_freeDecompressionContext(). +* Provides a pointer to a fully allocated and initialized LizardF_decompressionContext object. +* Object can later be released using LizardF_freeDecompressionContext(). * @return : if != 0, there was an error during context creation. */ -LIZF_errorCode_t LIZF_createDecompressionContext(LIZF_decompressionContext_t* LIZF_decompressionContextPtr, unsigned versionNumber) +LizardF_errorCode_t LizardF_createDecompressionContext(LizardF_decompressionContext_t* LizardF_decompressionContextPtr, unsigned versionNumber) { - LIZF_dctx_t* const dctxPtr = (LIZF_dctx_t*)ALLOCATOR(1, sizeof(LIZF_dctx_t)); - if (dctxPtr==NULL) return (LIZF_errorCode_t)-LIZF_ERROR_GENERIC; + LizardF_dctx_t* const dctxPtr = (LizardF_dctx_t*)ALLOCATOR(1, sizeof(LizardF_dctx_t)); + if (dctxPtr==NULL) return (LizardF_errorCode_t)-LizardF_ERROR_GENERIC; dctxPtr->version = versionNumber; - *LIZF_decompressionContextPtr = (LIZF_decompressionContext_t)dctxPtr; - return LIZF_OK_NoError; + *LizardF_decompressionContextPtr = (LizardF_decompressionContext_t)dctxPtr; + return LizardF_OK_NoError; } -LIZF_errorCode_t LIZF_freeDecompressionContext(LIZF_decompressionContext_t LIZF_decompressionContext) +LizardF_errorCode_t LizardF_freeDecompressionContext(LizardF_decompressionContext_t LizardF_decompressionContext) { - LIZF_errorCode_t result = LIZF_OK_NoError; - LIZF_dctx_t* const dctxPtr = (LIZF_dctx_t*)LIZF_decompressionContext; + LizardF_errorCode_t result = LizardF_OK_NoError; + LizardF_dctx_t* const dctxPtr = (LizardF_dctx_t*)LizardF_decompressionContext; if (dctxPtr != NULL) { /* can accept NULL input, like free() */ - result = (LIZF_errorCode_t)dctxPtr->dStage; + result = (LizardF_errorCode_t)dctxPtr->dStage; FREEMEM(dctxPtr->tmpIn); FREEMEM(dctxPtr->tmpOutBuffer); FREEMEM(dctxPtr); @@ -722,20 +722,20 @@ typedef enum { dstage_getHeader=0, dstage_storeHeader, } dStage_t; -/*! LIZF_headerSize() : +/*! LizardF_headerSize() : * @return : size of frame header -* or an error code, which can be tested using LIZF_isError() +* or an error code, which can be tested using LizardF_isError() */ -static size_t LIZF_headerSize(const void* src, size_t srcSize) +static size_t LizardF_headerSize(const void* src, size_t srcSize) { /* minimal srcSize to determine header size */ - if (srcSize < 5) return (size_t)-LIZF_ERROR_frameHeader_incomplete; + if (srcSize < 5) return (size_t)-LizardF_ERROR_frameHeader_incomplete; /* special case : skippable frames */ - if ((LIZF_readLE32(src) & 0xFFFFFFF0U) == LIZF_MAGIC_SKIPPABLE_START) return 8; + if ((LizardF_readLE32(src) & 0xFFFFFFF0U) == LIZARDF_MAGIC_SKIPPABLE_START) return 8; /* control magic number */ - if (LIZF_readLE32(src) != LIZF_MAGICNUMBER) return (size_t)-LIZF_ERROR_frameType_unknown; + if (LizardF_readLE32(src) != LIZARDF_MAGICNUMBER) return (size_t)-LizardF_ERROR_frameType_unknown; /* Frame Header Size */ { BYTE const FLG = ((const BYTE*)src)[4]; @@ -745,15 +745,15 @@ static size_t LIZF_headerSize(const void* src, size_t srcSize) } -/*! LIZF_decodeHeader() : +/*! LizardF_decodeHeader() : input : `srcVoidPtr` points at the **beginning of the frame** output : set internal values of dctx, such as dctxPtr->frameInfo and dctxPtr->dStage. Also allocates internal buffers. @return : nb Bytes read from srcVoidPtr (necessarily <= srcSize) - or an error code (testable with LIZF_isError()) + or an error code (testable with LizardF_isError()) */ -static size_t LIZF_decodeHeader(LIZF_dctx_t* dctxPtr, const void* srcVoidPtr, size_t srcSize) +static size_t LizardF_decodeHeader(LizardF_dctx_t* dctxPtr, const void* srcVoidPtr, size_t srcSize) { BYTE FLG, BD, HC; unsigned version, blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, blockSizeID; @@ -762,12 +762,12 @@ static size_t LIZF_decodeHeader(LIZF_dctx_t* dctxPtr, const void* srcVoidPtr, si const BYTE* srcPtr = (const BYTE*)srcVoidPtr; /* need to decode header to get frameInfo */ - if (srcSize < minFHSize) return (size_t)-LIZF_ERROR_frameHeader_incomplete; /* minimal frame header size */ + if (srcSize < minFHSize) return (size_t)-LizardF_ERROR_frameHeader_incomplete; /* minimal frame header size */ memset(&(dctxPtr->frameInfo), 0, sizeof(dctxPtr->frameInfo)); /* special case : skippable frames */ - if ((LIZF_readLE32(srcPtr) & 0xFFFFFFF0U) == LIZF_MAGIC_SKIPPABLE_START) { - dctxPtr->frameInfo.frameType = LIZF_skippableFrame; + if ((LizardF_readLE32(srcPtr) & 0xFFFFFFF0U) == LIZARDF_MAGIC_SKIPPABLE_START) { + dctxPtr->frameInfo.frameType = LizardF_skippableFrame; if (srcVoidPtr == (void*)(dctxPtr->header)) { dctxPtr->tmpInSize = srcSize; dctxPtr->tmpInTarget = 8; @@ -780,8 +780,8 @@ static size_t LIZF_decodeHeader(LIZF_dctx_t* dctxPtr, const void* srcVoidPtr, si } /* control magic number */ - if (LIZF_readLE32(srcPtr) != LIZF_MAGICNUMBER) return (size_t)-LIZF_ERROR_frameType_unknown; - dctxPtr->frameInfo.frameType = LIZF_frame; + if (LizardF_readLE32(srcPtr) != LIZARDF_MAGICNUMBER) return (size_t)-LizardF_ERROR_frameType_unknown; + dctxPtr->frameInfo.frameType = LizardF_frame; /* Flags */ FLG = srcPtr[4]; @@ -808,40 +808,39 @@ static size_t LIZF_decodeHeader(LIZF_dctx_t* dctxPtr, const void* srcVoidPtr, si blockSizeID = (BD>>4) & _3BITS; /* validate */ - if (version != 1) return (size_t)-LIZF_ERROR_headerVersion_wrong; /* Version Number, only supported value */ - if (blockChecksumFlag != 0) return (size_t)-LIZF_ERROR_blockChecksum_unsupported; /* Not supported for the time being */ - if (((FLG>>0)&_2BITS) != 0) return (size_t)-LIZF_ERROR_reservedFlag_set; /* Reserved bits */ - if (((BD>>7)&_1BIT) != 0) return (size_t)-LIZF_ERROR_reservedFlag_set; /* Reserved bit */ - if (blockSizeID < 1) return (size_t)-LIZF_ERROR_maxBlockSize_invalid; /* 1-7 only supported values for the time being */ - if (((BD>>0)&_4BITS) != 0) return (size_t)-LIZF_ERROR_reservedFlag_set; /* Reserved bits */ + if (version != 1) return (size_t)-LizardF_ERROR_headerVersion_wrong; /* Version Number, only supported value */ + if (blockChecksumFlag != 0) return (size_t)-LizardF_ERROR_blockChecksum_unsupported; /* Not supported for the time being */ + if (((FLG>>0)&_2BITS) != 0) return (size_t)-LizardF_ERROR_reservedFlag_set; /* Reserved bits */ + if (((BD>>7)&_1BIT) != 0) return (size_t)-LizardF_ERROR_reservedFlag_set; /* Reserved bit */ + if (blockSizeID < 1) return (size_t)-LizardF_ERROR_maxBlockSize_invalid; /* 1-7 only supported values for the time being */ + if (((BD>>0)&_4BITS) != 0) return (size_t)-LizardF_ERROR_reservedFlag_set; /* Reserved bits */ /* check */ - HC = LIZF_headerChecksum(srcPtr+4, frameHeaderSize-5); - if (HC != srcPtr[frameHeaderSize-1]) return (size_t)-LIZF_ERROR_headerChecksum_invalid; /* Bad header checksum error */ + HC = LizardF_headerChecksum(srcPtr+4, frameHeaderSize-5); + if (HC != srcPtr[frameHeaderSize-1]) return (size_t)-LizardF_ERROR_headerChecksum_invalid; /* Bad header checksum error */ /* save */ - dctxPtr->frameInfo.blockMode = (LIZF_blockMode_t)blockMode; - dctxPtr->frameInfo.contentChecksumFlag = (LIZF_contentChecksum_t)contentChecksumFlag; - dctxPtr->frameInfo.blockSizeID = (LIZF_blockSizeID_t)blockSizeID; + dctxPtr->frameInfo.blockMode = (LizardF_blockMode_t)blockMode; + dctxPtr->frameInfo.contentChecksumFlag = (LizardF_contentChecksum_t)contentChecksumFlag; + dctxPtr->frameInfo.blockSizeID = (LizardF_blockSizeID_t)blockSizeID; currentBlockSize = dctxPtr->maxBlockSize; - dctxPtr->maxBlockSize = LIZF_getBlockSize(blockSizeID); + dctxPtr->maxBlockSize = LizardF_getBlockSize(blockSizeID); if (contentSizeFlag) - dctxPtr->frameRemainingSize = dctxPtr->frameInfo.contentSize = LIZF_readLE64(srcPtr+6); + dctxPtr->frameRemainingSize = dctxPtr->frameInfo.contentSize = LizardF_readLE64(srcPtr+6); /* init */ if (contentChecksumFlag) XXH32_reset(&(dctxPtr->xxh), 0); /* alloc */ - bufferNeeded = dctxPtr->maxBlockSize + ((dctxPtr->frameInfo.blockMode==LIZF_blockLinked) * 2 * LIZ_DICT_SIZE); -// printf("LIZF_decodeHeader maxBlockSize=%d/%d bufferNeeded=%d/%d LIZF_blockLinked=%d\n", (int)dctxPtr->maxBlockSize, (int)dctxPtr->tmpInSize, (int)bufferNeeded, (int)dctxPtr->maxBufferSize, (dctxPtr->frameInfo.blockMode == LIZF_blockLinked)); + bufferNeeded = dctxPtr->maxBlockSize + ((dctxPtr->frameInfo.blockMode==LizardF_blockLinked) * 2 * LIZARD_DICT_SIZE); if (bufferNeeded > dctxPtr->maxBufferSize || dctxPtr->maxBlockSize > currentBlockSize) { /* tmp buffers too small */ FREEMEM(dctxPtr->tmpIn); FREEMEM(dctxPtr->tmpOutBuffer); dctxPtr->maxBufferSize = 0; dctxPtr->tmpIn = (BYTE*)ALLOCATOR(1, dctxPtr->maxBlockSize); - if (dctxPtr->tmpIn == NULL) return (size_t)-LIZF_ERROR_GENERIC; + if (dctxPtr->tmpIn == NULL) return (size_t)-LizardF_ERROR_GENERIC; dctxPtr->tmpOutBuffer= (BYTE*)ALLOCATOR(1, bufferNeeded); - if (dctxPtr->tmpOutBuffer== NULL) return (size_t)-LIZF_ERROR_GENERIC; + if (dctxPtr->tmpOutBuffer== NULL) return (size_t)-LizardF_ERROR_GENERIC; dctxPtr->maxBufferSize = bufferNeeded; } dctxPtr->tmpInSize = 0; @@ -858,36 +857,36 @@ static size_t LIZF_decodeHeader(LIZF_dctx_t* dctxPtr, const void* srcVoidPtr, si } -/*! LIZF_getFrameInfo() : +/*! LizardF_getFrameInfo() : * Decodes frame header information, such as blockSize. -* It is optional : you could start by calling directly LIZF_decompress() instead. +* It is optional : you could start by calling directly LizardF_decompress() instead. * The objective is to extract header information without starting decompression, typically for allocation purposes. -* LIZF_getFrameInfo() can also be used *after* starting decompression, on a valid LIZF_decompressionContext_t. +* LizardF_getFrameInfo() can also be used *after* starting decompression, on a valid LizardF_decompressionContext_t. * The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). * You are expected to resume decompression from where it stopped (srcBuffer + *srcSizePtr) -* @return : hint of the better `srcSize` to use for next call to LIZF_decompress, -* or an error code which can be tested using LIZF_isError(). +* @return : hint of the better `srcSize` to use for next call to LizardF_decompress, +* or an error code which can be tested using LizardF_isError(). */ -LIZF_errorCode_t LIZF_getFrameInfo(LIZF_decompressionContext_t dCtx, LIZF_frameInfo_t* frameInfoPtr, +LizardF_errorCode_t LizardF_getFrameInfo(LizardF_decompressionContext_t dCtx, LizardF_frameInfo_t* frameInfoPtr, const void* srcBuffer, size_t* srcSizePtr) { - LIZF_dctx_t* dctxPtr = (LIZF_dctx_t*)dCtx; + LizardF_dctx_t* dctxPtr = (LizardF_dctx_t*)dCtx; if (dctxPtr->dStage > dstage_storeHeader) { /* note : requires dstage_* header related to be at beginning of enum */ /* frameInfo already decoded */ size_t o=0, i=0; *srcSizePtr = 0; *frameInfoPtr = dctxPtr->frameInfo; - return LIZF_decompress(dCtx, NULL, &o, NULL, &i, NULL); /* returns : recommended nb of bytes for LIZF_decompress() */ + return LizardF_decompress(dCtx, NULL, &o, NULL, &i, NULL); /* returns : recommended nb of bytes for LizardF_decompress() */ } else { size_t nextSrcSize, o=0; - size_t const hSize = LIZF_headerSize(srcBuffer, *srcSizePtr); - if (LIZF_isError(hSize)) { *srcSizePtr=0; return hSize; } - if (*srcSizePtr < hSize) { *srcSizePtr=0; return (size_t)-LIZF_ERROR_frameHeader_incomplete; } + size_t const hSize = LizardF_headerSize(srcBuffer, *srcSizePtr); + if (LizardF_isError(hSize)) { *srcSizePtr=0; return hSize; } + if (*srcSizePtr < hSize) { *srcSizePtr=0; return (size_t)-LizardF_ERROR_frameHeader_incomplete; } *srcSizePtr = hSize; - nextSrcSize = LIZF_decompress(dCtx, NULL, &o, srcBuffer, srcSizePtr, NULL); - if (dctxPtr->dStage <= dstage_storeHeader) return (size_t)-LIZF_ERROR_frameHeader_incomplete; /* should not happen, already checked */ + nextSrcSize = LizardF_decompress(dCtx, NULL, &o, srcBuffer, srcSizePtr, NULL); + if (dctxPtr->dStage <= dstage_storeHeader) return (size_t)-LizardF_ERROR_frameHeader_incomplete; /* should not happen, already checked */ *frameInfoPtr = dctxPtr->frameInfo; return nextSrcSize; } @@ -895,14 +894,14 @@ LIZF_errorCode_t LIZF_getFrameInfo(LIZF_decompressionContext_t dCtx, LIZF_frameI /* trivial redirector, for common prototype */ -static int LIZF_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize) +static int LizardF_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize) { (void)dictStart; (void)dictSize; - return LIZ_decompress_safe (source, dest, compressedSize, maxDecompressedSize); + return Lizard_decompress_safe (source, dest, compressedSize, maxDecompressedSize); } -static void LIZF_updateDict(LIZF_dctx_t* dctxPtr, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp) +static void LizardF_updateDict(LizardF_dctx_t* dctxPtr, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp) { if (dctxPtr->dictSize==0) dctxPtr->dict = (const BYTE*)dstPtr; /* priority to dictionary continuity */ @@ -912,7 +911,7 @@ static void LIZF_updateDict(LIZF_dctx_t* dctxPtr, const BYTE* dstPtr, size_t dst return; } - if (dstPtr - dstPtr0 + dstSize >= LIZ_DICT_SIZE) { /* dstBuffer large enough to become dictionary */ + if (dstPtr - dstPtr0 + dstSize >= LIZARD_DICT_SIZE) { /* dstBuffer large enough to become dictionary */ dctxPtr->dict = (const BYTE*)dstPtr0; dctxPtr->dictSize = dstPtr - dstPtr0 + dstSize; return; @@ -926,9 +925,9 @@ static void LIZF_updateDict(LIZF_dctx_t* dctxPtr, const BYTE* dstPtr, size_t dst if (withinTmp) { /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */ size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; - size_t copySize = LIZ_DICT_SIZE - dctxPtr->tmpOutSize; + size_t copySize = LIZARD_DICT_SIZE - dctxPtr->tmpOutSize; const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart; - if (dctxPtr->tmpOutSize > LIZ_DICT_SIZE) copySize = 0; + if (dctxPtr->tmpOutSize > LIZARD_DICT_SIZE) copySize = 0; if (copySize > preserveSize) copySize = preserveSize; memcpy(dctxPtr->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); @@ -940,7 +939,7 @@ static void LIZF_updateDict(LIZF_dctx_t* dctxPtr, const BYTE* dstPtr, size_t dst if (dctxPtr->dict == dctxPtr->tmpOutBuffer) { /* copy dst into tmp to complete dict */ if (dctxPtr->dictSize + dstSize > dctxPtr->maxBufferSize) { /* tmp buffer not large enough */ - size_t preserveSize = LIZ_DICT_SIZE - dstSize; /* note : dstSize < LIZ_DICT_SIZE */ + size_t preserveSize = LIZARD_DICT_SIZE - dstSize; /* note : dstSize < LIZARD_DICT_SIZE */ memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize); dctxPtr->dictSize = preserveSize; } @@ -950,7 +949,7 @@ static void LIZF_updateDict(LIZF_dctx_t* dctxPtr, const BYTE* dstPtr, size_t dst } /* join dict & dest into tmp */ - { size_t preserveSize = LIZ_DICT_SIZE - dstSize; /* note : dstSize < LIZ_DICT_SIZE */ + { size_t preserveSize = LIZARD_DICT_SIZE - dstSize; /* note : dstSize < LIZARD_DICT_SIZE */ if (preserveSize > dctxPtr->dictSize) preserveSize = dctxPtr->dictSize; memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize); memcpy(dctxPtr->tmpOutBuffer + preserveSize, dstPtr, dstSize); @@ -961,7 +960,7 @@ static void LIZF_updateDict(LIZF_dctx_t* dctxPtr, const BYTE* dstPtr, size_t dst -/*! LIZF_decompress() : +/*! LizardF_decompress() : * Call this function repetitively to regenerate data compressed within srcBuffer. * The function will attempt to decode *srcSizePtr from srcBuffer, into dstBuffer of maximum size *dstSizePtr. * @@ -971,20 +970,20 @@ static void LIZF_updateDict(LIZF_dctx_t* dctxPtr, const BYTE* dstPtr, size_t dst * If the number of bytes read is < number of bytes provided, then the decompression operation is not complete. * You will have to call it again, continuing from where it stopped. * -* The function result is an hint of the better srcSize to use for next call to LIZF_decompress. +* The function result is an hint of the better srcSize to use for next call to LizardF_decompress. * Basically, it's the size of the current (or remaining) compressed block + header of next block. * Respecting the hint provides some boost to performance, since it allows less buffer shuffling. * Note that this is just a hint, you can always provide any srcSize you want. * When a frame is fully decoded, the function result will be 0. -* If decompression failed, function result is an error code which can be tested using LIZF_isError(). +* If decompression failed, function result is an error code which can be tested using LizardF_isError(). */ -size_t LIZF_decompress(LIZF_decompressionContext_t decompressionContext, +size_t LizardF_decompress(LizardF_decompressionContext_t decompressionContext, void* dstBuffer, size_t* dstSizePtr, const void* srcBuffer, size_t* srcSizePtr, - const LIZF_decompressOptions_t* decompressOptionsPtr) + const LizardF_decompressOptions_t* decompressOptionsPtr) { - LIZF_dctx_t* dctxPtr = (LIZF_dctx_t*)decompressionContext; - LIZF_decompressOptions_t optionsNull; + LizardF_dctx_t* dctxPtr = (LizardF_dctx_t*)decompressionContext; + LizardF_decompressOptions_t optionsNull; const BYTE* const srcStart = (const BYTE*)srcBuffer; const BYTE* const srcEnd = srcStart + *srcSizePtr; const BYTE* srcPtr = srcStart; @@ -1003,7 +1002,7 @@ size_t LIZF_decompress(LIZF_decompressionContext_t decompressionContext, /* expect to continue decoding src buffer where it left previously */ if (dctxPtr->srcExpect != NULL) { - if (srcStart != dctxPtr->srcExpect) return (size_t)-LIZF_ERROR_srcPtr_wrong; + if (srcStart != dctxPtr->srcExpect) return (size_t)-LizardF_ERROR_srcPtr_wrong; } /* programmed as a state machine */ @@ -1015,8 +1014,8 @@ size_t LIZF_decompress(LIZF_decompressionContext_t decompressionContext, case dstage_getHeader: if ((size_t)(srcEnd-srcPtr) >= maxFHSize) { /* enough to decode - shortcut */ - LIZF_errorCode_t const hSize = LIZF_decodeHeader(dctxPtr, srcPtr, srcEnd-srcPtr); - if (LIZF_isError(hSize)) return hSize; + LizardF_errorCode_t const hSize = LizardF_decodeHeader(dctxPtr, srcPtr, srcEnd-srcPtr); + if (LizardF_isError(hSize)) return hSize; srcPtr += hSize; break; } @@ -1036,8 +1035,8 @@ size_t LIZF_decompress(LIZF_decompressionContext_t decompressionContext, doAnotherStage = 0; /* not enough src data, ask for some more */ break; } - { LIZF_errorCode_t const hSize = LIZF_decodeHeader(dctxPtr, dctxPtr->header, dctxPtr->tmpInTarget); - if (LIZF_isError(hSize)) return hSize; + { LizardF_errorCode_t const hSize = LizardF_decodeHeader(dctxPtr, dctxPtr->header, dctxPtr->tmpInTarget); + if (LizardF_isError(hSize)) return hSize; } break; } @@ -1069,14 +1068,14 @@ size_t LIZF_decompress(LIZF_decompressionContext_t decompressionContext, } /* case dstage_decodeCBlockSize: */ /* no more direct access, to prevent scan-build warning */ - { size_t const nextCBlockSize = LIZF_readLE32(selectedIn) & 0x7FFFFFFFU; + { size_t const nextCBlockSize = LizardF_readLE32(selectedIn) & 0x7FFFFFFFU; if (nextCBlockSize==0) { /* frameEnd signal, no more CBlock */ dctxPtr->dStage = dstage_getSuffix; break; } - if (nextCBlockSize > dctxPtr->maxBlockSize) return (size_t)-LIZF_ERROR_GENERIC; /* invalid cBlockSize */ + if (nextCBlockSize > dctxPtr->maxBlockSize) return (size_t)-LizardF_ERROR_GENERIC; /* invalid cBlockSize */ dctxPtr->tmpInTarget = nextCBlockSize; - if (LIZF_readLE32(selectedIn) & LIZF_BLOCKUNCOMPRESSED_FLAG) { + if (LizardF_readLE32(selectedIn) & LIZARDF_BLOCKUNCOMPRESSED_FLAG) { dctxPtr->dStage = dstage_copyDirect; break; } @@ -1097,8 +1096,8 @@ size_t LIZF_decompress(LIZF_decompressionContext_t decompressionContext, if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= sizeToCopy; /* dictionary management */ - if (dctxPtr->frameInfo.blockMode==LIZF_blockLinked) - LIZF_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 0); + if (dctxPtr->frameInfo.blockMode==LizardF_blockLinked) + LizardF_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 0); srcPtr += sizeToCopy; dstPtr += sizeToCopy; @@ -1150,19 +1149,19 @@ size_t LIZF_decompress(LIZF_decompressionContext_t decompressionContext, { int (*decoder)(const char*, char*, int, int, const char*, int); int decodedSize; - if (dctxPtr->frameInfo.blockMode == LIZF_blockLinked) - decoder = LIZ_decompress_safe_usingDict; + if (dctxPtr->frameInfo.blockMode == LizardF_blockLinked) + decoder = Lizard_decompress_safe_usingDict; else - decoder = LIZF_decompress_safe; + decoder = LizardF_decompress_safe; decodedSize = decoder((const char*)selectedIn, (char*)dstPtr, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); - if (decodedSize < 0) return (size_t)-LIZF_ERROR_GENERIC; /* decompression failed */ + if (decodedSize < 0) return (size_t)-LizardF_ERROR_GENERIC; /* decompression failed */ if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dstPtr, decodedSize); if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize; /* dictionary management */ - if (dctxPtr->frameInfo.blockMode==LIZF_blockLinked) - LIZF_updateDict(dctxPtr, dstPtr, decodedSize, dstStart, 0); + if (dctxPtr->frameInfo.blockMode==LizardF_blockLinked) + LizardF_updateDict(dctxPtr, dstPtr, decodedSize, dstStart, 0); dstPtr += decodedSize; dctxPtr->dStage = dstage_getCBlockSize; @@ -1174,29 +1173,29 @@ size_t LIZF_decompress(LIZF_decompressionContext_t decompressionContext, { int (*decoder)(const char*, char*, int, int, const char*, int); int decodedSize; - if (dctxPtr->frameInfo.blockMode == LIZF_blockLinked) - decoder = LIZ_decompress_safe_usingDict; + if (dctxPtr->frameInfo.blockMode == LizardF_blockLinked) + decoder = Lizard_decompress_safe_usingDict; else - decoder = LIZF_decompress_safe; + decoder = LizardF_decompress_safe; /* ensure enough place for tmpOut */ - if (dctxPtr->frameInfo.blockMode == LIZF_blockLinked) { + if (dctxPtr->frameInfo.blockMode == LizardF_blockLinked) { if (dctxPtr->dict == dctxPtr->tmpOutBuffer) { - if (dctxPtr->dictSize > 2 * LIZ_DICT_SIZE) { - memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - LIZ_DICT_SIZE, LIZ_DICT_SIZE); - dctxPtr->dictSize = LIZ_DICT_SIZE; + if (dctxPtr->dictSize > 2 * LIZARD_DICT_SIZE) { + memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - LIZARD_DICT_SIZE, LIZARD_DICT_SIZE); + dctxPtr->dictSize = LIZARD_DICT_SIZE; } dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + dctxPtr->dictSize; } else { /* dict not within tmp */ size_t reservedDictSpace = dctxPtr->dictSize; - if (reservedDictSpace > LIZ_DICT_SIZE) reservedDictSpace = LIZ_DICT_SIZE; + if (reservedDictSpace > LIZARD_DICT_SIZE) reservedDictSpace = LIZARD_DICT_SIZE; dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + reservedDictSpace; } } /* Decode */ decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); - if (decodedSize < 0) return (size_t)-LIZF_ERROR_decompressionFailed; /* decompression failed */ + if (decodedSize < 0) return (size_t)-LizardF_ERROR_decompressionFailed; /* decompression failed */ if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize); if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize; dctxPtr->tmpOutSize = decodedSize; @@ -1211,8 +1210,8 @@ size_t LIZF_decompress(LIZF_decompressionContext_t decompressionContext, memcpy(dstPtr, dctxPtr->tmpOut + dctxPtr->tmpOutStart, sizeToCopy); /* dictionary management */ - if (dctxPtr->frameInfo.blockMode==LIZF_blockLinked) - LIZF_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 1); + if (dctxPtr->frameInfo.blockMode==LizardF_blockLinked) + LizardF_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 1); dctxPtr->tmpOutStart += sizeToCopy; dstPtr += sizeToCopy; @@ -1229,7 +1228,7 @@ size_t LIZF_decompress(LIZF_decompressionContext_t decompressionContext, case dstage_getSuffix: { size_t const suffixSize = dctxPtr->frameInfo.contentChecksumFlag * 4; - if (dctxPtr->frameRemainingSize) return (size_t)-LIZF_ERROR_frameSize_wrong; /* incorrect frame size decoded */ + if (dctxPtr->frameRemainingSize) return (size_t)-LizardF_ERROR_frameSize_wrong; /* incorrect frame size decoded */ if (suffixSize == 0) { /* frame completed */ nextSrcSizeHint = 0; dctxPtr->dStage = dstage_getHeader; @@ -1262,9 +1261,9 @@ size_t LIZF_decompress(LIZF_decompressionContext_t decompressionContext, } /* case dstage_checkSuffix: */ /* no direct call, to avoid scan-build warning */ - { U32 const readCRC = LIZF_readLE32(selectedIn); + { U32 const readCRC = LizardF_readLE32(selectedIn); U32 const resultCRC = XXH32_digest(&(dctxPtr->xxh)); - if (readCRC != resultCRC) return (size_t)-LIZF_ERROR_contentChecksum_invalid; + if (readCRC != resultCRC) return (size_t)-LizardF_ERROR_contentChecksum_invalid; nextSrcSizeHint = 0; dctxPtr->dStage = dstage_getHeader; doAnotherStage = 0; @@ -1299,7 +1298,7 @@ size_t LIZF_decompress(LIZF_decompressionContext_t decompressionContext, } /* case dstage_decodeSFrameSize: */ /* no direct access */ - { size_t const SFrameSize = LIZF_readLE32(selectedIn); + { size_t const SFrameSize = LizardF_readLE32(selectedIn); dctxPtr->frameInfo.contentSize = SFrameSize; dctxPtr->tmpInTarget = SFrameSize; dctxPtr->dStage = dstage_skipSkippable; @@ -1321,7 +1320,7 @@ size_t LIZF_decompress(LIZF_decompressionContext_t decompressionContext, } /* preserve dictionary within tmp if necessary */ - if ( (dctxPtr->frameInfo.blockMode==LIZF_blockLinked) + if ( (dctxPtr->frameInfo.blockMode==LizardF_blockLinked) &&(dctxPtr->dict != dctxPtr->tmpOutBuffer) &&(!decompressOptionsPtr->stableDst) &&((unsigned)(dctxPtr->dStage-1) < (unsigned)(dstage_getSuffix-1)) @@ -1329,9 +1328,9 @@ size_t LIZF_decompress(LIZF_decompressionContext_t decompressionContext, { if (dctxPtr->dStage == dstage_flushOut) { size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; - size_t copySize = LIZ_DICT_SIZE - dctxPtr->tmpOutSize; + size_t copySize = LIZARD_DICT_SIZE - dctxPtr->tmpOutSize; const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart; - if (dctxPtr->tmpOutSize > LIZ_DICT_SIZE) copySize = 0; + if (dctxPtr->tmpOutSize > LIZARD_DICT_SIZE) copySize = 0; if (copySize > preserveSize) copySize = preserveSize; memcpy(dctxPtr->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); @@ -1341,7 +1340,7 @@ size_t LIZF_decompress(LIZF_decompressionContext_t decompressionContext, } else { size_t newDictSize = dctxPtr->dictSize; const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize; - if ((newDictSize) > LIZ_DICT_SIZE) newDictSize = LIZ_DICT_SIZE; + if ((newDictSize) > LIZARD_DICT_SIZE) newDictSize = LIZARD_DICT_SIZE; memcpy(dctxPtr->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize); diff --git a/C/lizard/lizframe.h b/C/lizard/lizard_frame.h similarity index 51% rename from C/lizard/lizframe.h rename to C/lizard/lizard_frame.h index d743d101..7cdd2006 100644 --- a/C/lizard/lizframe.h +++ b/C/lizard/lizard_frame.h @@ -1,7 +1,9 @@ /* - LZ5 auto-framing library + Lizard auto-framing library Header File - Copyright (C) 2011-2015, Yann Collet. + Copyright (C) 2011-2015, Yann Collet + Copyright (C) 2016-2017, Przemyslaw Skibinski + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -28,13 +30,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - LZ5 source repository : https://github.com/inikep/lz5 + - Lizard source repository : https://github.com/inikep/lizard */ -/* LZ5F is a stand-alone API to create LZ5-compressed frames +/* LizardF is a stand-alone API to create Lizard-compressed frames * conformant with specification v1.5.1. * All related operations, including memory management, are handled internally by the library. - * You don't need lz5_compress.h when using lz5frame.h. + * You don't need lizard_compress.h when using lizard_frame.h. * */ #pragma once @@ -52,171 +54,171 @@ extern "C" { /*-************************************ * Error management **************************************/ -typedef size_t LIZF_errorCode_t; +typedef size_t LizardF_errorCode_t; -unsigned LIZF_isError(LIZF_errorCode_t code); -const char* LIZF_getErrorName(LIZF_errorCode_t code); /* return error code string; useful for debugging */ +unsigned LizardF_isError(LizardF_errorCode_t code); +const char* LizardF_getErrorName(LizardF_errorCode_t code); /* return error code string; useful for debugging */ /*-************************************ * Frame compression types **************************************/ -//#define LIZF_DISABLE_OBSOLETE_ENUMS -#ifndef LIZF_DISABLE_OBSOLETE_ENUMS -# define LIZF_OBSOLETE_ENUM(x) ,x +//#define LIZARDF_DISABLE_OBSOLETE_ENUMS +#ifndef LIZARDF_DISABLE_OBSOLETE_ENUMS +# define LIZARDF_OBSOLETE_ENUM(x) ,x #else -# define LIZF_OBSOLETE_ENUM(x) +# define LIZARDF_OBSOLETE_ENUM(x) #endif typedef enum { - LIZF_default=0, - LIZF_max128KB=1, - LIZF_max256KB=2, - LIZF_max1MB=3, - LIZF_max4MB=4, - LIZF_max16MB=5, - LIZF_max64MB=6, - LIZF_max256MB=7 -} LIZF_blockSizeID_t; + LizardF_default=0, + LizardF_max128KB=1, + LizardF_max256KB=2, + LizardF_max1MB=3, + LizardF_max4MB=4, + LizardF_max16MB=5, + LizardF_max64MB=6, + LizardF_max256MB=7 +} LizardF_blockSizeID_t; typedef enum { - LIZF_blockLinked=0, - LIZF_blockIndependent - LIZF_OBSOLETE_ENUM(blockLinked = LIZF_blockLinked) - LIZF_OBSOLETE_ENUM(blockIndependent = LIZF_blockIndependent) -} LIZF_blockMode_t; + LizardF_blockLinked=0, + LizardF_blockIndependent + LIZARDF_OBSOLETE_ENUM(blockLinked = LizardF_blockLinked) + LIZARDF_OBSOLETE_ENUM(blockIndependent = LizardF_blockIndependent) +} LizardF_blockMode_t; typedef enum { - LIZF_noContentChecksum=0, - LIZF_contentChecksumEnabled - LIZF_OBSOLETE_ENUM(noContentChecksum = LIZF_noContentChecksum) - LIZF_OBSOLETE_ENUM(contentChecksumEnabled = LIZF_contentChecksumEnabled) -} LIZF_contentChecksum_t; + LizardF_noContentChecksum=0, + LizardF_contentChecksumEnabled + LIZARDF_OBSOLETE_ENUM(noContentChecksum = LizardF_noContentChecksum) + LIZARDF_OBSOLETE_ENUM(contentChecksumEnabled = LizardF_contentChecksumEnabled) +} LizardF_contentChecksum_t; typedef enum { - LIZF_frame=0, - LIZF_skippableFrame - LIZF_OBSOLETE_ENUM(skippableFrame = LIZF_skippableFrame) -} LIZF_frameType_t; + LizardF_frame=0, + LizardF_skippableFrame + LIZARDF_OBSOLETE_ENUM(skippableFrame = LizardF_skippableFrame) +} LizardF_frameType_t; -#ifndef LIZF_DISABLE_OBSOLETE_ENUMS -typedef LIZF_blockSizeID_t blockSizeID_t; -typedef LIZF_blockMode_t blockMode_t; -typedef LIZF_frameType_t frameType_t; -typedef LIZF_contentChecksum_t contentChecksum_t; +#ifndef LIZARDF_DISABLE_OBSOLETE_ENUMS +typedef LizardF_blockSizeID_t blockSizeID_t; +typedef LizardF_blockMode_t blockMode_t; +typedef LizardF_frameType_t frameType_t; +typedef LizardF_contentChecksum_t contentChecksum_t; #endif typedef struct { - LIZF_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ - LIZF_blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */ - LIZF_contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */ - LIZF_frameType_t frameType; /* LIZF_frame, skippableFrame ; 0 == default */ + LizardF_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ + LizardF_blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */ + LizardF_contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */ + LizardF_frameType_t frameType; /* LizardF_frame, skippableFrame ; 0 == default */ unsigned long long contentSize; /* Size of uncompressed (original) content ; 0 == unknown */ unsigned reserved[2]; /* must be zero for forward compatibility */ -} LIZF_frameInfo_t; +} LizardF_frameInfo_t; typedef struct { - LIZF_frameInfo_t frameInfo; + LizardF_frameInfo_t frameInfo; int compressionLevel; /* 0 == default (fast mode); values above 16 count as 16; values below 0 count as 0 */ unsigned autoFlush; /* 1 == always flush (reduce need for tmp buffer) */ unsigned reserved[4]; /* must be zero for forward compatibility */ -} LIZF_preferences_t; +} LizardF_preferences_t; /*-********************************* * Simple compression function ***********************************/ -size_t LIZF_compressFrameBound(size_t srcSize, const LIZF_preferences_t* preferencesPtr); +size_t LizardF_compressFrameBound(size_t srcSize, const LizardF_preferences_t* preferencesPtr); -/*!LIZF_compressFrame() : - * Compress an entire srcBuffer into a valid LZ5 frame, as defined by specification v1.5.1 +/*!LizardF_compressFrame() : + * Compress an entire srcBuffer into a valid Lizard frame, as defined by specification v1.5.1 * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. - * You can get the minimum value of dstMaxSize by using LIZF_compressFrameBound() - * If this condition is not respected, LIZF_compressFrame() will fail (result is an errorCode) - * The LIZF_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. + * You can get the minimum value of dstMaxSize by using LizardF_compressFrameBound() + * If this condition is not respected, LizardF_compressFrame() will fail (result is an errorCode) + * The LizardF_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. * The result of the function is the number of bytes written into dstBuffer. - * The function outputs an error code if it fails (can be tested using LIZF_isError()) + * The function outputs an error code if it fails (can be tested using LizardF_isError()) */ -size_t LIZF_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LIZF_preferences_t* preferencesPtr); +size_t LizardF_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LizardF_preferences_t* preferencesPtr); /*-*********************************** * Advanced compression functions *************************************/ -typedef struct LIZF_cctx_s* LIZF_compressionContext_t; /* must be aligned on 8-bytes */ +typedef struct LizardF_cctx_s* LizardF_compressionContext_t; /* must be aligned on 8-bytes */ typedef struct { - unsigned stableSrc; /* 1 == src content will remain available on future calls to LIZF_compress(); avoid saving src content within tmp buffer as future dictionary */ + unsigned stableSrc; /* 1 == src content will remain available on future calls to LizardF_compress(); avoid saving src content within tmp buffer as future dictionary */ unsigned reserved[3]; -} LIZF_compressOptions_t; +} LizardF_compressOptions_t; /* Resource Management */ -#define LIZF_VERSION 100 -LIZF_errorCode_t LIZF_createCompressionContext(LIZF_compressionContext_t* cctxPtr, unsigned version); -LIZF_errorCode_t LIZF_freeCompressionContext(LIZF_compressionContext_t cctx); -/* LIZF_createCompressionContext() : +#define LIZARDF_VERSION 100 +LizardF_errorCode_t LizardF_createCompressionContext(LizardF_compressionContext_t* cctxPtr, unsigned version); +LizardF_errorCode_t LizardF_freeCompressionContext(LizardF_compressionContext_t cctx); +/* LizardF_createCompressionContext() : * The first thing to do is to create a compressionContext object, which will be used in all compression operations. - * This is achieved using LIZF_createCompressionContext(), which takes as argument a version and an LIZF_preferences_t structure. - * The version provided MUST be LIZF_VERSION. It is intended to track potential version differences between different binaries. - * The function will provide a pointer to a fully allocated LIZF_compressionContext_t object. - * If the result LIZF_errorCode_t is not zero, there was an error during context creation. - * Object can release its memory using LIZF_freeCompressionContext(); + * This is achieved using LizardF_createCompressionContext(), which takes as argument a version and an LizardF_preferences_t structure. + * The version provided MUST be LIZARDF_VERSION. It is intended to track potential version differences between different binaries. + * The function will provide a pointer to a fully allocated LizardF_compressionContext_t object. + * If the result LizardF_errorCode_t is not zero, there was an error during context creation. + * Object can release its memory using LizardF_freeCompressionContext(); */ /* Compression */ -size_t LIZF_compressBegin(LIZF_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LIZF_preferences_t* prefsPtr); -/* LIZF_compressBegin() : +size_t LizardF_compressBegin(LizardF_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LizardF_preferences_t* prefsPtr); +/* LizardF_compressBegin() : * will write the frame header into dstBuffer. * dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is 15 bytes. - * The LIZF_preferences_t structure is optional : you can provide NULL as argument, all preferences will then be set to default. + * The LizardF_preferences_t structure is optional : you can provide NULL as argument, all preferences will then be set to default. * The result of the function is the number of bytes written into dstBuffer for the header - * or an error code (can be tested using LIZF_isError()) + * or an error code (can be tested using LizardF_isError()) */ -size_t LIZF_compressBound(size_t srcSize, const LIZF_preferences_t* prefsPtr); -/* LIZF_compressBound() : +size_t LizardF_compressBound(size_t srcSize, const LizardF_preferences_t* prefsPtr); +/* LizardF_compressBound() : * Provides the minimum size of Dst buffer given srcSize to handle worst case situations. * Different preferences can produce different results. * prefsPtr is optional : you can provide NULL as argument, all preferences will then be set to cover worst case. * This function includes frame termination cost (4 bytes, or 8 if frame checksum is enabled) */ -size_t LIZF_compressUpdate(LIZF_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LIZF_compressOptions_t* cOptPtr); -/* LIZF_compressUpdate() - * LIZF_compressUpdate() can be called repetitively to compress as much data as necessary. +size_t LizardF_compressUpdate(LizardF_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LizardF_compressOptions_t* cOptPtr); +/* LizardF_compressUpdate() + * LizardF_compressUpdate() can be called repetitively to compress as much data as necessary. * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. - * You can get the minimum value of dstMaxSize by using LIZF_compressBound(). - * If this condition is not respected, LIZF_compress() will fail (result is an errorCode). - * LIZF_compressUpdate() doesn't guarantee error recovery, so you have to reset compression context when an error occurs. - * The LIZF_compressOptions_t structure is optional : you can provide NULL as argument. + * You can get the minimum value of dstMaxSize by using LizardF_compressBound(). + * If this condition is not respected, LizardF_compress() will fail (result is an errorCode). + * LizardF_compressUpdate() doesn't guarantee error recovery, so you have to reset compression context when an error occurs. + * The LizardF_compressOptions_t structure is optional : you can provide NULL as argument. * The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered. - * The function outputs an error code if it fails (can be tested using LIZF_isError()) + * The function outputs an error code if it fails (can be tested using LizardF_isError()) */ -size_t LIZF_flush(LIZF_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LIZF_compressOptions_t* cOptPtr); -/* LIZF_flush() +size_t LizardF_flush(LizardF_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LizardF_compressOptions_t* cOptPtr); +/* LizardF_flush() * Should you need to generate compressed data immediately, without waiting for the current block to be filled, - * you can call LIZ_flush(), which will immediately compress any remaining data buffered within cctx. + * you can call Lizard_flush(), which will immediately compress any remaining data buffered within cctx. * Note that dstMaxSize must be large enough to ensure the operation will be successful. - * LIZF_compressOptions_t structure is optional : you can provide NULL as argument. + * LizardF_compressOptions_t structure is optional : you can provide NULL as argument. * The result of the function is the number of bytes written into dstBuffer * (it can be zero, this means there was no data left within cctx) - * The function outputs an error code if it fails (can be tested using LIZF_isError()) + * The function outputs an error code if it fails (can be tested using LizardF_isError()) */ -size_t LIZF_compressEnd(LIZF_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LIZF_compressOptions_t* cOptPtr); -/* LIZF_compressEnd() - * When you want to properly finish the compressed frame, just call LIZF_compressEnd(). - * It will flush whatever data remained within compressionContext (like LIZ_flush()) +size_t LizardF_compressEnd(LizardF_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LizardF_compressOptions_t* cOptPtr); +/* LizardF_compressEnd() + * When you want to properly finish the compressed frame, just call LizardF_compressEnd(). + * It will flush whatever data remained within compressionContext (like Lizard_flush()) * but also properly finalize the frame, with an endMark and a checksum. * The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled) - * The function outputs an error code if it fails (can be tested using LIZF_isError()) - * The LIZF_compressOptions_t structure is optional : you can provide NULL as argument. - * A successful call to LIZF_compressEnd() makes cctx available again for next compression task. + * The function outputs an error code if it fails (can be tested using LizardF_isError()) + * The LizardF_compressOptions_t structure is optional : you can provide NULL as argument. + * A successful call to LizardF_compressEnd() makes cctx available again for next compression task. */ @@ -224,48 +226,48 @@ size_t LIZF_compressEnd(LIZF_compressionContext_t cctx, void* dstBuffer, size_t * Decompression functions ***********************************/ -typedef struct LIZF_dctx_s* LIZF_decompressionContext_t; /* must be aligned on 8-bytes */ +typedef struct LizardF_dctx_s* LizardF_decompressionContext_t; /* must be aligned on 8-bytes */ typedef struct { unsigned stableDst; /* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */ unsigned reserved[3]; -} LIZF_decompressOptions_t; +} LizardF_decompressOptions_t; /* Resource management */ -/*!LIZF_createDecompressionContext() : - * Create an LIZF_decompressionContext_t object, which will be used to track all decompression operations. - * The version provided MUST be LIZF_VERSION. It is intended to track potential breaking differences between different versions. - * The function will provide a pointer to a fully allocated and initialized LIZF_decompressionContext_t object. - * The result is an errorCode, which can be tested using LIZF_isError(). - * dctx memory can be released using LIZF_freeDecompressionContext(); - * The result of LIZF_freeDecompressionContext() is indicative of the current state of decompressionContext when being released. +/*!LizardF_createDecompressionContext() : + * Create an LizardF_decompressionContext_t object, which will be used to track all decompression operations. + * The version provided MUST be LIZARDF_VERSION. It is intended to track potential breaking differences between different versions. + * The function will provide a pointer to a fully allocated and initialized LizardF_decompressionContext_t object. + * The result is an errorCode, which can be tested using LizardF_isError(). + * dctx memory can be released using LizardF_freeDecompressionContext(); + * The result of LizardF_freeDecompressionContext() is indicative of the current state of decompressionContext when being released. * That is, it should be == 0 if decompression has been completed fully and correctly. */ -LIZF_errorCode_t LIZF_createDecompressionContext(LIZF_decompressionContext_t* dctxPtr, unsigned version); -LIZF_errorCode_t LIZF_freeDecompressionContext(LIZF_decompressionContext_t dctx); +LizardF_errorCode_t LizardF_createDecompressionContext(LizardF_decompressionContext_t* dctxPtr, unsigned version); +LizardF_errorCode_t LizardF_freeDecompressionContext(LizardF_decompressionContext_t dctx); /*====== Decompression ======*/ -/*!LIZF_getFrameInfo() : +/*!LizardF_getFrameInfo() : * This function decodes frame header information (such as max blockSize, frame checksum, etc.). * Its usage is optional. The objective is to extract frame header information, typically for allocation purposes. * A header size is variable and can be from 7 to 15 bytes. It's also possible to input more bytes than that. * The number of bytes read from srcBuffer will be updated within *srcSizePtr (necessarily <= original value). - * (note that LIZF_getFrameInfo() can also be used anytime *after* starting decompression, in this case 0 input byte is enough) - * Frame header info is *copied into* an already allocated LIZF_frameInfo_t structure. - * The function result is an hint about how many srcSize bytes LIZF_decompress() expects for next call, - * or an error code which can be tested using LIZF_isError() + * (note that LizardF_getFrameInfo() can also be used anytime *after* starting decompression, in this case 0 input byte is enough) + * Frame header info is *copied into* an already allocated LizardF_frameInfo_t structure. + * The function result is an hint about how many srcSize bytes LizardF_decompress() expects for next call, + * or an error code which can be tested using LizardF_isError() * (typically, when there is not enough src bytes to fully decode the frame header) * Decompression is expected to resume from where it stopped (srcBuffer + *srcSizePtr) */ -size_t LIZF_getFrameInfo(LIZF_decompressionContext_t dctx, - LIZF_frameInfo_t* frameInfoPtr, +size_t LizardF_getFrameInfo(LizardF_decompressionContext_t dctx, + LizardF_frameInfo_t* frameInfoPtr, const void* srcBuffer, size_t* srcSizePtr); -/*!LIZF_decompress() : +/*!LizardF_decompress() : * Call this function repetitively to regenerate data compressed within srcBuffer. * The function will attempt to decode *srcSizePtr bytes from srcBuffer, into dstBuffer of maximum size *dstSizePtr. * @@ -274,25 +276,25 @@ size_t LIZF_getFrameInfo(LIZF_decompressionContext_t dctx, * The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). * If number of bytes read is < number of bytes provided, then decompression operation is not completed. * It typically happens when dstBuffer is not large enough to contain all decoded data. - * LIZF_decompress() must be called again, starting from where it stopped (srcBuffer + *srcSizePtr) + * LizardF_decompress() must be called again, starting from where it stopped (srcBuffer + *srcSizePtr) * The function will check this condition, and refuse to continue if it is not respected. * * `dstBuffer` is expected to be flushed between each call to the function, its content will be overwritten. * `dst` arguments can be changed at will at each consecutive call to the function. * - * The function result is an hint of how many `srcSize` bytes LIZF_decompress() expects for next call. + * The function result is an hint of how many `srcSize` bytes LizardF_decompress() expects for next call. * Schematically, it's the size of the current (or remaining) compressed block + header of next block. * Respecting the hint provides some boost to performance, since it does skip intermediate buffers. * This is just a hint though, it's always possible to provide any srcSize. * When a frame is fully decoded, the function result will be 0 (no more data expected). - * If decompression failed, function result is an error code, which can be tested using LIZF_isError(). + * If decompression failed, function result is an error code, which can be tested using LizardF_isError(). * * After a frame is fully decoded, dctx can be used again to decompress another frame. */ -size_t LIZF_decompress(LIZF_decompressionContext_t dctx, +size_t LizardF_decompress(LizardF_decompressionContext_t dctx, void* dstBuffer, size_t* dstSizePtr, const void* srcBuffer, size_t* srcSizePtr, - const LIZF_decompressOptions_t* dOptPtr); + const LizardF_decompressOptions_t* dOptPtr); diff --git a/C/lizard/lizframe_static.h b/C/lizard/lizard_frame_static.h similarity index 80% rename from C/lizard/lizframe_static.h rename to C/lizard/lizard_frame_static.h index bfebcf16..da807560 100644 --- a/C/lizard/lizframe_static.h +++ b/C/lizard/lizard_frame_static.h @@ -1,7 +1,8 @@ /* - LZ5 auto-framing library + Lizard auto-framing library Header File for static linking only Copyright (C) 2011-2015, Yann Collet. + Copyright (C) 2016-2017, Przemyslaw Skibinski BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) @@ -29,7 +30,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - LZ5 source repository : https://github.com/inikep/lz5 + - Lizard source repository : https://github.com/inikep/lizard */ #pragma once @@ -38,7 +39,7 @@ extern "C" { #endif -/* lz5frame_static.h should be used solely in the context of static linking. +/* lizard_frame_static.h should be used solely in the context of static linking. * It contains definitions which may still change overtime. * Never use it in the context of DLL linking. * */ @@ -47,13 +48,13 @@ extern "C" { /************************************** * Includes **************************************/ -#include "lizframe.h" +#include "lizard_frame.h" /************************************** * Error management * ************************************/ -#define LIZF_LIST_ERRORS(ITEM) \ +#define LIZARDF_LIST_ERRORS(ITEM) \ ITEM(OK_NoError) ITEM(ERROR_GENERIC) \ ITEM(ERROR_maxBlockSize_invalid) ITEM(ERROR_blockMode_invalid) ITEM(ERROR_contentChecksumFlag_invalid) \ ITEM(ERROR_compressionLevel_invalid) \ @@ -66,13 +67,13 @@ extern "C" { ITEM(ERROR_headerChecksum_invalid) ITEM(ERROR_contentChecksum_invalid) \ ITEM(ERROR_maxCode) -//#define LIZF_DISABLE_OLD_ENUMS -#ifndef LIZF_DISABLE_OLD_ENUMS -#define LIZF_GENERATE_ENUM(ENUM) LIZF_##ENUM, ENUM = LIZF_##ENUM, +//#define LIZARDF_DISABLE_OLD_ENUMS +#ifndef LIZARDF_DISABLE_OLD_ENUMS +#define LIZARDF_GENERATE_ENUM(ENUM) LizardF_##ENUM, ENUM = LizardF_##ENUM, #else -#define LIZF_GENERATE_ENUM(ENUM) LIZF_##ENUM, +#define LIZARDF_GENERATE_ENUM(ENUM) LizardF_##ENUM, #endif -typedef enum { LIZF_LIST_ERRORS(LIZF_GENERATE_ENUM) } LIZF_errorCodes; /* enum is exposed, to handle specific errors; compare function result to -enum value */ +typedef enum { LIZARDF_LIST_ERRORS(LIZARDF_GENERATE_ENUM) } LizardF_errorCodes; /* enum is exposed, to handle specific errors; compare function result to -enum value */ #if defined (__cplusplus) diff --git a/C/lizard/lizard_parser_fast.h b/C/lizard/lizard_parser_fast.h new file mode 100644 index 00000000..f9e54b7f --- /dev/null +++ b/C/lizard/lizard_parser_fast.h @@ -0,0 +1,196 @@ +#define LIZARD_FAST_MIN_OFFSET 8 +#define LIZARD_FAST_LONGOFF_MM 0 /* not used with offsets > 1<<16 */ + +/************************************** +* Hash Functions +**************************************/ +static size_t Lizard_hashPosition(const void* p) +{ + if (MEM_64bits()) + return Lizard_hash5Ptr(p, LIZARD_HASHLOG_LZ4); + return Lizard_hash4Ptr(p, LIZARD_HASHLOG_LZ4); +} + +static void Lizard_putPositionOnHash(const BYTE* p, size_t h, U32* hashTable, const BYTE* srcBase) +{ + hashTable[h] = (U32)(p-srcBase); +} + +static void Lizard_putPosition(const BYTE* p, U32* hashTable, const BYTE* srcBase) +{ + size_t const h = Lizard_hashPosition(p); + Lizard_putPositionOnHash(p, h, hashTable, srcBase); +} + +static U32 Lizard_getPositionOnHash(size_t h, U32* hashTable) +{ + return hashTable[h]; +} + +static U32 Lizard_getPosition(const BYTE* p, U32* hashTable) +{ + size_t const h = Lizard_hashPosition(p); + return Lizard_getPositionOnHash(h, hashTable); +} + + +static const U32 Lizard_skipTrigger = 6; /* Increase this value ==> compression run slower on incompressible data */ +static const U32 Lizard_minLength = (MFLIMIT+1); + + +FORCE_INLINE int Lizard_compress_fast( + Lizard_stream_t* const ctx, + const BYTE* ip, + const BYTE* const iend) +{ + const U32 acceleration = 1; + const BYTE* base = ctx->base; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const lowPrefixPtr = base + dictLimit; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + const BYTE* anchor = ip; + + size_t forwardH, matchIndex; + const U32 maxDistance = (1 << ctx->params.windowLog) - 1; + const U32 lowLimit = (ctx->lowLimit + maxDistance >= (U32)(ip - base)) ? ctx->lowLimit : (U32)(ip - base) - maxDistance; + + /* Init conditions */ + if ((U32)(iend-ip) > (U32)LIZARD_MAX_INPUT_SIZE) goto _output_error; /* Unsupported inputSize, too large (or negative) */ + + if ((U32)(iend-ip) < Lizard_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ + + /* First Byte */ + Lizard_putPosition(ip, ctx->hashTable, base); + ip++; forwardH = Lizard_hashPosition(ip); + + /* Main Loop */ + for ( ; ; ) { + const BYTE* match; + size_t matchLength; + + /* Find a match */ + { const BYTE* forwardIp = ip; + unsigned step = 1; + unsigned searchMatchNb = acceleration << Lizard_skipTrigger; + while (1) { + size_t const h = forwardH; + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> Lizard_skipTrigger); + + if (unlikely(forwardIp > mflimit)) goto _last_literals; + + matchIndex = Lizard_getPositionOnHash(h, ctx->hashTable); + forwardH = Lizard_hashPosition(forwardIp); + Lizard_putPositionOnHash(ip, h, ctx->hashTable, base); + + if ((matchIndex < lowLimit) || (matchIndex >= (U32)(ip - base)) || (base + matchIndex + maxDistance < ip)) continue; + + if (matchIndex >= dictLimit) { + match = base + matchIndex; +#if LIZARD_FAST_MIN_OFFSET > 0 + if ((U32)(ip - match) >= LIZARD_FAST_MIN_OFFSET) +#endif + if (MEM_read32(match) == MEM_read32(ip)) + { + int back = 0; + matchLength = Lizard_count(ip+MINMATCH, match+MINMATCH, matchlimit); + + while ((ip+back > anchor) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; + matchLength -= back; +#if LIZARD_FAST_LONGOFF_MM > 0 + if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) +#endif + { + ip += back; + match += back; + break; + } + } + } else { + match = dictBase + matchIndex; +#if LIZARD_FAST_MIN_OFFSET > 0 + if ((U32)(ip - (base + matchIndex)) >= LIZARD_FAST_MIN_OFFSET) +#endif + if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ + if (MEM_read32(match) == MEM_read32(ip)) { + const U32 newLowLimit = (lowLimit + maxDistance >= (U32)(ip-base)) ? lowLimit : (U32)(ip - base) - maxDistance; + int back = 0; + matchLength = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr); + + while ((ip+back > anchor) && (matchIndex+back > newLowLimit) && (ip[back-1] == match[back-1])) back--; + matchLength -= back; + match = base + matchIndex + back; +#if LIZARD_FAST_LONGOFF_MM > 0 + if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) +#endif + { + ip += back; + break; + } + } + } + } // while (1) + } + +_next_match: + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, matchLength+MINMATCH, match)) goto _output_error; + + /* Test end of chunk */ + if (ip > mflimit) break; + + /* Fill table */ + Lizard_putPosition(ip-2, ctx->hashTable, base); + + /* Test next position */ + matchIndex = Lizard_getPosition(ip, ctx->hashTable); + Lizard_putPosition(ip, ctx->hashTable, base); + if ((matchIndex >= lowLimit) && (matchIndex < (U32)(ip - base)) && (base + matchIndex + maxDistance >= ip)) + { + if (matchIndex >= dictLimit) { + match = base + matchIndex; +#if LIZARD_FAST_MIN_OFFSET > 0 + if ((U32)(ip - match) >= LIZARD_FAST_MIN_OFFSET) +#endif + if (MEM_read32(match) == MEM_read32(ip)) + { + matchLength = Lizard_count(ip+MINMATCH, match+MINMATCH, matchlimit); +#if LIZARD_FAST_LONGOFF_MM > 0 + if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) +#endif + goto _next_match; + } + } else { + match = dictBase + matchIndex; +#if LIZARD_FAST_MIN_OFFSET > 0 + if ((U32)(ip - (base + matchIndex)) >= LIZARD_FAST_MIN_OFFSET) +#endif + if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ + if (MEM_read32(match) == MEM_read32(ip)) { + matchLength = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr); + match = base + matchIndex; +#if LIZARD_FAST_LONGOFF_MM > 0 + if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) +#endif + goto _next_match; + } + } + } + + /* Prepare next loop */ + forwardH = Lizard_hashPosition(++ip); + } + +_last_literals: + /* Encode Last Literals */ + ip = iend; + if (Lizard_encodeLastLiterals_LZ4(ctx, &ip, &anchor)) goto _output_error; + + /* End */ + return 1; +_output_error: + return 0; +} diff --git a/C/lizard/lizard_parser_fastbig.h b/C/lizard/lizard_parser_fastbig.h new file mode 100644 index 00000000..30c464fc --- /dev/null +++ b/C/lizard/lizard_parser_fastbig.h @@ -0,0 +1,175 @@ +#define LIZARD_FASTBIG_LONGOFF_MM MM_LONGOFF + +/************************************** +* Hash Functions +**************************************/ +static size_t Lizard_hashPositionHLog(const void* p, int hashLog) +{ + if (MEM_64bits()) + return Lizard_hash5Ptr(p, hashLog); + return Lizard_hash4Ptr(p, hashLog); +} + +static void Lizard_putPositionOnHashHLog(const BYTE* p, size_t h, U32* hashTable, const BYTE* srcBase) +{ + hashTable[h] = (U32)(p-srcBase); +} + +static void Lizard_putPositionHLog(const BYTE* p, U32* hashTable, const BYTE* srcBase, int hashLog) +{ + size_t const h = Lizard_hashPositionHLog(p, hashLog); + Lizard_putPositionOnHashHLog(p, h, hashTable, srcBase); +} + +static U32 Lizard_getPositionOnHashHLog(size_t h, U32* hashTable) +{ + return hashTable[h]; +} + +static U32 Lizard_getPositionHLog(const BYTE* p, U32* hashTable, int hashLog) +{ + size_t const h = Lizard_hashPositionHLog(p, hashLog); + return Lizard_getPositionOnHashHLog(h, hashTable); +} + +FORCE_INLINE int Lizard_compress_fastBig( + Lizard_stream_t* const ctx, + const BYTE* ip, + const BYTE* const iend) +{ + const U32 acceleration = 1; + const BYTE* base = ctx->base; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const lowPrefixPtr = base + dictLimit; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + const BYTE* anchor = ip; + + size_t forwardH, matchIndex; + const int hashLog = ctx->params.hashLog; + const U32 maxDistance = (1 << ctx->params.windowLog) - 1; + const U32 lowLimit = (ctx->lowLimit + maxDistance >= (U32)(ip - base)) ? ctx->lowLimit : (U32)(ip - base) - maxDistance; + + /* Init conditions */ + if ((U32)(iend-ip) > (U32)LIZARD_MAX_INPUT_SIZE) goto _output_error; /* Unsupported inputSize, too large (or negative) */ + + if ((U32)(iend-ip) < Lizard_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ + + /* First Byte */ + Lizard_putPositionHLog(ip, ctx->hashTable, base, hashLog); + ip++; forwardH = Lizard_hashPositionHLog(ip, hashLog); + + /* Main Loop */ + for ( ; ; ) { + const BYTE* match; + size_t matchLength; + + /* Find a match */ + { const BYTE* forwardIp = ip; + unsigned step = 1; + unsigned searchMatchNb = acceleration << Lizard_skipTrigger; + while (1) { + size_t const h = forwardH; + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> Lizard_skipTrigger); + + if (unlikely(forwardIp > mflimit)) goto _last_literals; + + matchIndex = Lizard_getPositionOnHashHLog(h, ctx->hashTable); + forwardH = Lizard_hashPositionHLog(forwardIp, hashLog); + Lizard_putPositionOnHashHLog(ip, h, ctx->hashTable, base); + + if ((matchIndex < lowLimit) || (matchIndex >= (U32)(ip - base)) || (base + matchIndex + maxDistance < ip)) continue; + + if (matchIndex >= dictLimit) { + match = base + matchIndex; + if ((U32)(ip - match) >= LIZARD_FAST_MIN_OFFSET) + if (MEM_read32(match) == MEM_read32(ip)) + { + int back = 0; + matchLength = Lizard_count(ip+MINMATCH, match+MINMATCH, matchlimit); + + while ((ip+back > anchor) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; + matchLength -= back; + if ((matchLength >= LIZARD_FASTBIG_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) + { + ip += back; + match += back; + break; + } + } + } else { + match = dictBase + matchIndex; + if ((U32)(ip - (base + matchIndex)) >= LIZARD_FAST_MIN_OFFSET) + if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ + if (MEM_read32(match) == MEM_read32(ip)) { + const U32 newLowLimit = (lowLimit + maxDistance >= (U32)(ip-base)) ? lowLimit : (U32)(ip - base) - maxDistance; + int back = 0; + matchLength = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr); + + while ((ip+back > anchor) && (matchIndex+back > newLowLimit) && (ip[back-1] == match[back-1])) back--; + matchLength -= back; + match = base + matchIndex + back; + if ((matchLength >= LIZARD_FASTBIG_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) + { + ip += back; + break; + } + } + } + } // while (1) + } + +_next_match: + if (Lizard_encodeSequence_LIZv1(ctx, &ip, &anchor, matchLength+MINMATCH, match)) goto _output_error; + + /* Test end of chunk */ + if (ip > mflimit) break; + + /* Fill table */ + Lizard_putPositionHLog(ip-2, ctx->hashTable, base, hashLog); + + /* Test next position */ + matchIndex = Lizard_getPositionHLog(ip, ctx->hashTable, hashLog); + Lizard_putPositionHLog(ip, ctx->hashTable, base, hashLog); + if ((matchIndex >= lowLimit) && (matchIndex < (U32)(ip - base)) && (base + matchIndex + maxDistance >= ip)) + { + if (matchIndex >= dictLimit) { + match = base + matchIndex; + if ((U32)(ip - match) >= LIZARD_FAST_MIN_OFFSET) + if (MEM_read32(match) == MEM_read32(ip)) + { + matchLength = Lizard_count(ip+MINMATCH, match+MINMATCH, matchlimit); + if ((matchLength >= LIZARD_FASTBIG_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) + goto _next_match; + } + } else { + match = dictBase + matchIndex; + if ((U32)(ip - (base + matchIndex)) >= LIZARD_FAST_MIN_OFFSET) + if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ + if (MEM_read32(match) == MEM_read32(ip)) { + matchLength = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr); + match = base + matchIndex; + if ((matchLength >= LIZARD_FASTBIG_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) + goto _next_match; + } + } + } + + /* Prepare next loop */ + forwardH = Lizard_hashPositionHLog(++ip, hashLog); + } + +_last_literals: + /* Encode Last Literals */ + ip = iend; + if (Lizard_encodeLastLiterals_LIZv1(ctx, &ip, &anchor)) goto _output_error; + + /* End */ + return 1; +_output_error: + return 0; +} diff --git a/C/lizard/lizard_parser_fastsmall.h b/C/lizard/lizard_parser_fastsmall.h new file mode 100644 index 00000000..550edc5b --- /dev/null +++ b/C/lizard/lizard_parser_fastsmall.h @@ -0,0 +1,189 @@ +/************************************** +* Hash Functions +**************************************/ +static size_t Lizard_hashPositionSmall(const void* p) +{ + if (MEM_64bits()) + return Lizard_hash5Ptr(p, LIZARD_HASHLOG_LZ4SM); + return Lizard_hash4Ptr(p, LIZARD_HASHLOG_LZ4SM); +} + +static void Lizard_putPositionOnHashSmall(const BYTE* p, size_t h, U32* hashTable, const BYTE* srcBase) +{ + hashTable[h] = (U32)(p-srcBase); +} + +static void Lizard_putPositionSmall(const BYTE* p, U32* hashTable, const BYTE* srcBase) +{ + size_t const h = Lizard_hashPositionSmall(p); + Lizard_putPositionOnHashSmall(p, h, hashTable, srcBase); +} + +static U32 Lizard_getPositionOnHashSmall(size_t h, U32* hashTable) +{ + return hashTable[h]; +} + +static U32 Lizard_getPositionSmall(const BYTE* p, U32* hashTable) +{ + size_t const h = Lizard_hashPositionSmall(p); + return Lizard_getPositionOnHashSmall(h, hashTable); +} + + +FORCE_INLINE int Lizard_compress_fastSmall( + Lizard_stream_t* const ctx, + const BYTE* ip, + const BYTE* const iend) +{ + const U32 acceleration = 1; + const BYTE* base = ctx->base; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const lowPrefixPtr = base + dictLimit; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + const BYTE* anchor = ip; + + size_t forwardH, matchIndex; + const U32 maxDistance = (1 << ctx->params.windowLog) - 1; + const U32 lowLimit = (ctx->lowLimit + maxDistance >= (U32)(ip - base)) ? ctx->lowLimit : (U32)(ip - base) - maxDistance; + + /* Init conditions */ + if ((U32)(iend-ip) > (U32)LIZARD_MAX_INPUT_SIZE) goto _output_error; /* Unsupported inputSize, too large (or negative) */ + + if ((U32)(iend-ip) < Lizard_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ + + /* First Byte */ + Lizard_putPositionSmall(ip, ctx->hashTable, base); + ip++; forwardH = Lizard_hashPositionSmall(ip); + + /* Main Loop */ + for ( ; ; ) { + const BYTE* match; + size_t matchLength; + + /* Find a match */ + { const BYTE* forwardIp = ip; + unsigned step = 1; + unsigned searchMatchNb = acceleration << Lizard_skipTrigger; + while (1) { + size_t const h = forwardH; + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> Lizard_skipTrigger); + + if (unlikely(forwardIp > mflimit)) goto _last_literals; + + matchIndex = Lizard_getPositionOnHashSmall(h, ctx->hashTable); + forwardH = Lizard_hashPositionSmall(forwardIp); + Lizard_putPositionOnHashSmall(ip, h, ctx->hashTable, base); + + if ((matchIndex < lowLimit) || (matchIndex >= (U32)(ip - base)) || (base + matchIndex + maxDistance < ip)) continue; + + if (matchIndex >= dictLimit) { + match = base + matchIndex; +#if LIZARD_FAST_MIN_OFFSET > 0 + if ((U32)(ip - match) >= LIZARD_FAST_MIN_OFFSET) +#endif + if (MEM_read32(match) == MEM_read32(ip)) + { + int back = 0; + matchLength = Lizard_count(ip+MINMATCH, match+MINMATCH, matchlimit); + + while ((ip+back > anchor) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; + matchLength -= back; +#if LIZARD_FAST_LONGOFF_MM > 0 + if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) +#endif + { + ip += back; + match += back; + break; + } + } + } else { + match = dictBase + matchIndex; +#if LIZARD_FAST_MIN_OFFSET > 0 + if ((U32)(ip - (base + matchIndex)) >= LIZARD_FAST_MIN_OFFSET) +#endif + if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ + if (MEM_read32(match) == MEM_read32(ip)) { + const U32 newLowLimit = (lowLimit + maxDistance >= (U32)(ip-base)) ? lowLimit : (U32)(ip - base) - maxDistance; + int back = 0; + matchLength = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr); + + while ((ip+back > anchor) && (matchIndex+back > newLowLimit) && (ip[back-1] == match[back-1])) back--; + matchLength -= back; + match = base + matchIndex + back; +#if LIZARD_FAST_LONGOFF_MM > 0 + if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) +#endif + { + ip += back; + break; + } + } + } + } // while (1) + } + +_next_match: + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, matchLength+MINMATCH, match)) goto _output_error; + + /* Test end of chunk */ + if (ip > mflimit) break; + + /* Fill table */ + Lizard_putPositionSmall(ip-2, ctx->hashTable, base); + + /* Test next position */ + matchIndex = Lizard_getPositionSmall(ip, ctx->hashTable); + Lizard_putPositionSmall(ip, ctx->hashTable, base); + if ((matchIndex >= lowLimit) && (matchIndex < (U32)(ip - base)) && (base + matchIndex + maxDistance >= ip)) + { + if (matchIndex >= dictLimit) { + match = base + matchIndex; +#if LIZARD_FAST_MIN_OFFSET > 0 + if ((U32)(ip - match) >= LIZARD_FAST_MIN_OFFSET) +#endif + if (MEM_read32(match) == MEM_read32(ip)) + { + matchLength = Lizard_count(ip+MINMATCH, match+MINMATCH, matchlimit); +#if LIZARD_FAST_LONGOFF_MM > 0 + if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) +#endif + goto _next_match; + } + } else { + match = dictBase + matchIndex; +#if LIZARD_FAST_MIN_OFFSET > 0 + if ((U32)(ip - (base + matchIndex)) >= LIZARD_FAST_MIN_OFFSET) +#endif + if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ + if (MEM_read32(match) == MEM_read32(ip)) { + matchLength = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr); + match = base + matchIndex; +#if LIZARD_FAST_LONGOFF_MM > 0 + if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) +#endif + goto _next_match; + } + } + } + + /* Prepare next loop */ + forwardH = Lizard_hashPositionSmall(++ip); + } + +_last_literals: + /* Encode Last Literals */ + ip = iend; + if (Lizard_encodeLastLiterals_LZ4(ctx, &ip, &anchor)) goto _output_error; + + /* End */ + return 1; +_output_error: + return 0; +} diff --git a/C/lizard/liz_parser_hashchain.h b/C/lizard/lizard_parser_hashchain.h similarity index 71% rename from C/lizard/liz_parser_hashchain.h rename to C/lizard/lizard_parser_hashchain.h index 5af68199..64b9d76c 100644 --- a/C/lizard/liz_parser_hashchain.h +++ b/C/lizard/lizard_parser_hashchain.h @@ -1,16 +1,16 @@ -#define LIZ_HC_MIN_OFFSET 8 -#define LIZ_HC_LONGOFF_MM 0 /* not used with offsets > 1<<16 */ +#define LIZARD_HC_MIN_OFFSET 8 +#define LIZARD_HC_LONGOFF_MM 0 /* not used with offsets > 1<<16 */ #define OPTIMAL_ML (int)((ML_MASK_LZ4-1)+MINMATCH) #define GET_MINMATCH(offset) (MINMATCH) #if 1 - #define LIZ_HC_HASH_FUNCTION(ip, hashLog) LIZ_hashPtr(ip, hashLog, ctx->params.searchLength) + #define LIZARD_HC_HASH_FUNCTION(ip, hashLog) Lizard_hashPtr(ip, hashLog, ctx->params.searchLength) #else - #define LIZ_HC_HASH_FUNCTION(ip, hashLog) LIZ_hash5Ptr(ip, hashLog) + #define LIZARD_HC_HASH_FUNCTION(ip, hashLog) Lizard_hash5Ptr(ip, hashLog) #endif /* Update chains up to ip (excluded) */ -FORCE_INLINE void LIZ_Insert (LIZ_stream_t* ctx, const BYTE* ip) +FORCE_INLINE void Lizard_Insert (Lizard_stream_t* ctx, const BYTE* ip) { U32* const chainTable = ctx->chainTable; U32* const hashTable = ctx->hashTable; @@ -25,14 +25,14 @@ FORCE_INLINE void LIZ_Insert (LIZ_stream_t* ctx, const BYTE* ip) const U32 maxDistance = (1 << ctx->params.windowLog) - 1; while (idx < target) { - size_t const h = LIZ_hashPtr(base+idx, hashLog, ctx->params.searchLength); + size_t const h = Lizard_hashPtr(base+idx, hashLog, ctx->params.searchLength); size_t delta = idx - hashTable[h]; if (delta>maxDistance) delta = maxDistance; DELTANEXT(idx) = (U32)delta; - if (idx >= hashTable[h] + LIZ_HC_MIN_OFFSET) + if ((hashTable[h] >= idx) || (idx >= hashTable[h] + LIZARD_HC_MIN_OFFSET)) hashTable[h] = idx; #if MINMATCH == 3 - HashTable3[LIZ_hash3Ptr(base+idx, ctx->params.hashLog3)] = idx; + HashTable3[Lizard_hash3Ptr(base+idx, ctx->params.hashLog3)] = idx; #endif idx++; } @@ -42,7 +42,7 @@ FORCE_INLINE void LIZ_Insert (LIZ_stream_t* ctx, const BYTE* ip) -FORCE_INLINE int LIZ_InsertAndFindBestMatch (LIZ_stream_t* ctx, /* Index table will be updated */ +FORCE_INLINE int Lizard_InsertAndFindBestMatch (Lizard_stream_t* ctx, /* Index table will be updated */ const BYTE* ip, const BYTE* const iLimit, const BYTE** matchpos) { @@ -60,39 +60,39 @@ FORCE_INLINE int LIZ_InsertAndFindBestMatch (LIZ_stream_t* ctx, /* Index table const int hashLog = ctx->params.hashLog; const U32 contentMask = (1 << ctx->params.contentLog) - 1; const U32 maxDistance = (1 << ctx->params.windowLog) - 1; - const U32 lowLimit = (ctx->lowLimit + maxDistance >= (U32)(ip-base)) ? ctx->lowLimit : (U32)(ip - base) - maxDistance; + const U32 current = (U32)(ip - base); + const U32 lowLimit = (ctx->lowLimit + maxDistance >= current) ? ctx->lowLimit : current - maxDistance; /* HC4 match finder */ - LIZ_Insert(ctx, ip); - matchIndex = HashTable[LIZ_HC_HASH_FUNCTION(ip, hashLog)]; + Lizard_Insert(ctx, ip); + matchIndex = HashTable[LIZARD_HC_HASH_FUNCTION(ip, hashLog)]; - while ((matchIndex>=lowLimit) && (nbAttempts)) { + while ((matchIndex < current) && (matchIndex >= lowLimit) && (nbAttempts)) { nbAttempts--; if (matchIndex >= dictLimit) { match = base + matchIndex; -#if LIZ_HC_MIN_OFFSET > 0 - if ((U32)(ip - match) >= LIZ_HC_MIN_OFFSET) +#if LIZARD_HC_MIN_OFFSET > 0 + if ((U32)(ip - match) >= LIZARD_HC_MIN_OFFSET) #endif if (*(match+ml) == *(ip+ml) && (MEM_read32(match) == MEM_read32(ip))) { - size_t const mlt = LIZ_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; -#if LIZ_HC_LONGOFF_MM > 0 - if ((mlt >= LIZ_HC_LONGOFF_MM) || ((U32)(ip - match) < LIZ_MAX_16BIT_OFFSET)) + size_t const mlt = Lizard_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; +#if LIZARD_HC_LONGOFF_MM > 0 + if ((mlt >= LIZARD_HC_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) #endif if (mlt > ml) { ml = mlt; *matchpos = match; } } } else { match = dictBase + matchIndex; -// fprintf(stderr, "dictBase[%p]+matchIndex[%d]=match[%p] dictLimit=%d base=%p ip=%p iLimit=%p off=%d\n", dictBase, matchIndex, match, dictLimit, base, ip, iLimit, (U32)(ip-match)); -#if LIZ_HC_MIN_OFFSET > 0 - if ((U32)(ip - (base + matchIndex)) >= LIZ_HC_MIN_OFFSET) +#if LIZARD_HC_MIN_OFFSET > 0 + if ((U32)(ip - (base + matchIndex)) >= LIZARD_HC_MIN_OFFSET) #endif if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ if (MEM_read32(match) == MEM_read32(ip)) { - size_t mlt = LIZ_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH; -#if LIZ_HC_LONGOFF_MM > 0 - if ((mlt >= LIZ_HC_LONGOFF_MM) || ((U32)(ip - (base + matchIndex)) < LIZ_MAX_16BIT_OFFSET)) + size_t mlt = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH; +#if LIZARD_HC_LONGOFF_MM > 0 + if ((mlt >= LIZARD_HC_LONGOFF_MM) || ((U32)(ip - (base + matchIndex)) < LIZARD_MAX_16BIT_OFFSET)) #endif if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */ } @@ -106,8 +106,8 @@ FORCE_INLINE int LIZ_InsertAndFindBestMatch (LIZ_stream_t* ctx, /* Index table } -FORCE_INLINE int LIZ_InsertAndGetWiderMatch ( - LIZ_stream_t* ctx, +FORCE_INLINE int Lizard_InsertAndGetWiderMatch ( + Lizard_stream_t* ctx, const BYTE* const ip, const BYTE* const iLowLimit, const BYTE* const iHighLimit, @@ -128,28 +128,29 @@ FORCE_INLINE int LIZ_InsertAndGetWiderMatch ( const int hashLog = ctx->params.hashLog; const U32 contentMask = (1 << ctx->params.contentLog) - 1; const U32 maxDistance = (1 << ctx->params.windowLog) - 1; - const U32 lowLimit = (ctx->lowLimit + maxDistance >= (U32)(ip-base)) ? ctx->lowLimit : (U32)(ip - base) - maxDistance; - - /* First Match */ - LIZ_Insert(ctx, ip); - matchIndex = HashTable[LIZ_HC_HASH_FUNCTION(ip, hashLog)]; + const U32 current = (U32)(ip - base); + const U32 lowLimit = (ctx->lowLimit + maxDistance >= current) ? ctx->lowLimit : current - maxDistance; - while ((matchIndex>=lowLimit) && (nbAttempts)) { + /* First Match */ + Lizard_Insert(ctx, ip); + matchIndex = HashTable[LIZARD_HC_HASH_FUNCTION(ip, hashLog)]; + + while ((matchIndex < current) && (matchIndex >= lowLimit) && (nbAttempts)) { nbAttempts--; if (matchIndex >= dictLimit) { const BYTE* match = base + matchIndex; -#if LIZ_HC_MIN_OFFSET > 0 - if ((U32)(ip - match) >= LIZ_HC_MIN_OFFSET) +#if LIZARD_HC_MIN_OFFSET > 0 + if ((U32)(ip - match) >= LIZARD_HC_MIN_OFFSET) #endif if (*(iLowLimit + longest) == *(match - LLdelta + longest)) { if (MEM_read32(match) == MEM_read32(ip)) { - int mlt = MINMATCH + LIZ_count(ip+MINMATCH, match+MINMATCH, iHighLimit); + int mlt = MINMATCH + Lizard_count(ip+MINMATCH, match+MINMATCH, iHighLimit); int back = 0; while ((ip+back > iLowLimit) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; mlt -= back; -#if LIZ_HC_LONGOFF_MM > 0 - if ((mlt >= LIZ_HC_LONGOFF_MM) || ((U32)(ip - match) < LIZ_MAX_16BIT_OFFSET)) +#if LIZARD_HC_LONGOFF_MM > 0 + if ((mlt >= LIZARD_HC_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) #endif if (mlt > longest) { longest = (int)mlt; @@ -160,17 +161,17 @@ FORCE_INLINE int LIZ_InsertAndGetWiderMatch ( } } else { const BYTE* match = dictBase + matchIndex; -#if LIZ_HC_MIN_OFFSET > 0 - if ((U32)(ip - (base + matchIndex)) >= LIZ_HC_MIN_OFFSET) +#if LIZARD_HC_MIN_OFFSET > 0 + if ((U32)(ip - (base + matchIndex)) >= LIZARD_HC_MIN_OFFSET) #endif if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ if (MEM_read32(match) == MEM_read32(ip)) { int back=0; - size_t mlt = LIZ_count_2segments(ip+MINMATCH, match+MINMATCH, iHighLimit, dictEnd, lowPrefixPtr) + MINMATCH; + size_t mlt = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, iHighLimit, dictEnd, lowPrefixPtr) + MINMATCH; while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == match[back-1])) back--; mlt -= back; -#if LIZ_HC_LONGOFF_MM > 0 - if ((mlt >= LIZ_HC_LONGOFF_MM) || ((U32)(ip - (base + matchIndex)) < LIZ_MAX_16BIT_OFFSET)) +#if LIZARD_HC_LONGOFF_MM > 0 + if ((mlt >= LIZARD_HC_LONGOFF_MM) || ((U32)(ip - (base + matchIndex)) < LIZARD_MAX_16BIT_OFFSET)) #endif if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; } } @@ -184,8 +185,8 @@ FORCE_INLINE int LIZ_InsertAndGetWiderMatch ( } -FORCE_INLINE int LIZ_compress_hashChain ( - LIZ_stream_t* const ctx, +FORCE_INLINE int Lizard_compress_hashChain ( + Lizard_stream_t* const ctx, const BYTE* ip, const BYTE* const iend) { @@ -207,7 +208,7 @@ FORCE_INLINE int LIZ_compress_hashChain ( /* Main Loop */ while (ip < mflimit) { - ml = LIZ_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref)); + ml = Lizard_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref)); if (!ml) { ip++; continue; } /* saved, in case we would skip too much */ @@ -217,11 +218,11 @@ FORCE_INLINE int LIZ_compress_hashChain ( _Search2: if (ip+ml < mflimit) - ml2 = LIZ_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2); + ml2 = Lizard_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2); else ml2 = ml; if (ml2 == ml) { /* No better match */ - if (LIZ_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; continue; } @@ -254,7 +255,7 @@ _Search3: if (ip+new_ml > start2 + ml2 - GET_MINMATCH((U32)(start2 - ref2))) { new_ml = (int)(start2 - ip) + ml2 - GET_MINMATCH((U32)(start2 - ref2)); if (new_ml < GET_MINMATCH((U32)(ip - ref))) { // match2 doesn't fit - if (LIZ_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; continue; } } @@ -268,16 +269,16 @@ _Search3: /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */ if (start2 + ml2 < mflimit) - ml3 = LIZ_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3); + ml3 = Lizard_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3); else ml3 = ml2; if (ml3 == ml2) { /* No better match : 2 sequences to encode */ /* ip & ref are known; Now for ml */ if (start2 < ip+ml) ml = (int)(start2 - ip); /* Now, encode 2 sequences */ - if (LIZ_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; ip = start2; - if (LIZ_encodeSequence_LZ4(ctx, &ip, &anchor, ml2, ref2)) return 0; + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml2, ref2)) return 0; continue; } @@ -295,7 +296,7 @@ _Search3: } } - if (LIZ_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; ip = start3; ref = ref3; ml = ml3; @@ -323,7 +324,7 @@ _Search3: if (ip + ml > start2 + ml2 - GET_MINMATCH((U32)(start2 - ref2))) { ml = (int)(start2 - ip) + ml2 - GET_MINMATCH((U32)(start2 - ref2)); if (ml < GET_MINMATCH((U32)(ip - ref))) { // match2 doesn't fit, remove it - if (LIZ_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; ip = start3; ref = ref3; ml = ml3; @@ -344,7 +345,7 @@ _Search3: ml = (int)(start2 - ip); } } - if (LIZ_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; ip = start2; ref = ref2; @@ -359,7 +360,7 @@ _Search3: /* Encode Last Literals */ ip = iend; - if (LIZ_encodeLastLiterals_LZ4(ctx, &ip, &anchor)) goto _output_error; + if (Lizard_encodeLastLiterals_LZ4(ctx, &ip, &anchor)) goto _output_error; /* End */ return 1; diff --git a/C/lizard/liz_parser_lowestprice.h b/C/lizard/lizard_parser_lowestprice.h similarity index 65% rename from C/lizard/liz_parser_lowestprice.h rename to C/lizard/lizard_parser_lowestprice.h index 27b274fc..f8b95dad 100644 --- a/C/lizard/liz_parser_lowestprice.h +++ b/C/lizard/lizard_parser_lowestprice.h @@ -1,7 +1,7 @@ -#define LIZ_LOWESTPRICE_MIN_OFFSET 8 +#define LIZARD_LOWESTPRICE_MIN_OFFSET 8 -FORCE_INLINE size_t LIZ_more_profitable(LIZ_stream_t* const ctx, const BYTE *best_ip, size_t best_off, size_t best_common, const BYTE *ip, size_t off, size_t common, size_t literals, int last_off) +FORCE_INLINE size_t Lizard_more_profitable(Lizard_stream_t* const ctx, const BYTE *best_ip, size_t best_off, size_t best_common, const BYTE *ip, size_t off, size_t common, size_t literals, int last_off) { size_t sum; @@ -13,20 +13,20 @@ FORCE_INLINE size_t LIZ_more_profitable(LIZ_stream_t* const ctx, const BYTE *bes if ((int)off == last_off) off = 0; // rep code if ((int)best_off == last_off) best_off = 0; - return LIZ_get_price_LZ5v2(ctx, last_off, ip, ctx->off24pos, sum - common, (U32)off, common) <= LIZ_get_price_LZ5v2(ctx, last_off, best_ip, ctx->off24pos, sum - best_common, (U32)best_off, best_common); + return Lizard_get_price_LIZv1(ctx, last_off, ip, ctx->off24pos, sum - common, (U32)off, common) <= Lizard_get_price_LIZv1(ctx, last_off, best_ip, ctx->off24pos, sum - best_common, (U32)best_off, best_common); } -FORCE_INLINE size_t LIZ_better_price(LIZ_stream_t* const ctx, const BYTE *best_ip, size_t best_off, size_t best_common, const BYTE *ip, size_t off, size_t common, int last_off) +FORCE_INLINE size_t Lizard_better_price(Lizard_stream_t* const ctx, const BYTE *best_ip, size_t best_off, size_t best_common, const BYTE *ip, size_t off, size_t common, int last_off) { if ((int)off == last_off) off = 0; // rep code if ((int)best_off == last_off) best_off = 0; - return LIZ_get_price_LZ5v2(ctx, last_off, ip, ctx->off24pos, 0, (U32)off, common) < LIZ_get_price_LZ5v2(ctx, last_off, best_ip, ctx->off24pos, common - best_common, (U32)best_off, best_common); + return Lizard_get_price_LIZv1(ctx, last_off, ip, ctx->off24pos, 0, (U32)off, common) < Lizard_get_price_LIZv1(ctx, last_off, best_ip, ctx->off24pos, common - best_common, (U32)best_off, best_common); } -FORCE_INLINE int LIZ_FindMatchLowestPrice (LIZ_stream_t* ctx, /* Index table will be updated */ +FORCE_INLINE int Lizard_FindMatchLowestPrice (Lizard_stream_t* ctx, /* Index table will be updated */ const BYTE* ip, const BYTE* const iLimit, const BYTE** matchpos) { @@ -47,15 +47,15 @@ FORCE_INLINE int LIZ_FindMatchLowestPrice (LIZ_stream_t* ctx, /* Index table w int nbAttempts=ctx->params.searchNum; size_t ml=0, mlt; - matchIndex = HashTable[LIZ_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)]; + matchIndex = HashTable[Lizard_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)]; - if (ctx->last_off >= LIZ_LOWESTPRICE_MIN_OFFSET) { + if (ctx->last_off >= LIZARD_LOWESTPRICE_MIN_OFFSET) { intptr_t matchIndexLO = (ip - ctx->last_off) - base; if (matchIndexLO >= lowLimit) { if (matchIndexLO >= dictLimit) { match = base + matchIndexLO; - mlt = LIZ_count(ip, match, iLimit);// + MINMATCH; - // if ((mlt >= minMatchLongOff) || (ctx->last_off < LIZ_MAX_16BIT_OFFSET)) + mlt = Lizard_count(ip, match, iLimit);// + MINMATCH; + // if ((mlt >= minMatchLongOff) || (ctx->last_off < LIZARD_MAX_16BIT_OFFSET)) if (mlt > REPMINMATCH) { *matchpos = match; return (int)mlt; @@ -63,8 +63,8 @@ FORCE_INLINE int LIZ_FindMatchLowestPrice (LIZ_stream_t* ctx, /* Index table w } else { match = dictBase + matchIndexLO; if ((U32)((dictLimit-1) - matchIndexLO) >= 3) { /* intentional overflow */ - mlt = LIZ_count_2segments(ip, match, iLimit, dictEnd, lowPrefixPtr); - // if ((mlt >= minMatchLongOff) || (ctx->last_off < LIZ_MAX_16BIT_OFFSET)) + mlt = Lizard_count_2segments(ip, match, iLimit, dictEnd, lowPrefixPtr); + // if ((mlt >= minMatchLongOff) || (ctx->last_off < LIZARD_MAX_16BIT_OFFSET)) if (mlt > REPMINMATCH) { *matchpos = base + matchIndexLO; /* virtual matchpos */ return (int)mlt; @@ -77,16 +77,16 @@ FORCE_INLINE int LIZ_FindMatchLowestPrice (LIZ_stream_t* ctx, /* Index table w #if MINMATCH == 3 { - U32 matchIndex3 = ctx->hashTable3[LIZ_hash3Ptr(ip, ctx->params.hashLog3)]; + U32 matchIndex3 = ctx->hashTable3[Lizard_hash3Ptr(ip, ctx->params.hashLog3)]; if (matchIndex3 < current && matchIndex3 >= lowLimit) { size_t offset = (size_t)current - matchIndex3; - if (offset < LIZ_MAX_8BIT_OFFSET) + if (offset < LIZARD_MAX_8BIT_OFFSET) { match = ip - offset; if (match > base && MEM_readMINMATCH(ip) == MEM_readMINMATCH(match)) { - ml = 3;//LIZ_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; + ml = 3;//Lizard_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; *matchpos = match; } } @@ -96,22 +96,21 @@ FORCE_INLINE int LIZ_FindMatchLowestPrice (LIZ_stream_t* ctx, /* Index table w while ((matchIndex < current) && (matchIndex >= lowLimit) && (nbAttempts)) { nbAttempts--; match = base + matchIndex; - if ((U32)(ip - match) >= LIZ_LOWESTPRICE_MIN_OFFSET) { + if ((U32)(ip - match) >= LIZARD_LOWESTPRICE_MIN_OFFSET) { if (matchIndex >= dictLimit) { if (*(match+ml) == *(ip+ml) && (MEM_read32(match) == MEM_read32(ip))) { - mlt = LIZ_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; - if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZ_MAX_16BIT_OFFSET)) - if (!ml || (mlt > ml && LIZ_better_price(ctx, ip, (ip - *matchpos), ml, ip, (ip - match), mlt, ctx->last_off))) + mlt = Lizard_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; + if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) + if (!ml || (mlt > ml && Lizard_better_price(ctx, ip, (ip - *matchpos), ml, ip, (ip - match), mlt, ctx->last_off))) { ml = mlt; *matchpos = match; } } } else { matchDict = dictBase + matchIndex; - // fprintf(stderr, "dictBase[%p]+matchIndex[%d]=match[%p] dictLimit=%d base=%p ip=%p iLimit=%p off=%d\n", dictBase, matchIndex, match, dictLimit, base, ip, iLimit, (U32)(ip-match)); if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ if (MEM_read32(matchDict) == MEM_read32(ip)) { - mlt = LIZ_count_2segments(ip+MINMATCH, matchDict+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH; - if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZ_MAX_16BIT_OFFSET)) - if (!ml || (mlt > ml && LIZ_better_price(ctx, ip, (ip - *matchpos), ml, ip, (U32)(ip - match), mlt, ctx->last_off))) + mlt = Lizard_count_2segments(ip+MINMATCH, matchDict+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH; + if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) + if (!ml || (mlt > ml && Lizard_better_price(ctx, ip, (ip - *matchpos), ml, ip, (U32)(ip - match), mlt, ctx->last_off))) { ml = mlt; *matchpos = match; } /* virtual matchpos */ } } @@ -123,8 +122,8 @@ FORCE_INLINE int LIZ_FindMatchLowestPrice (LIZ_stream_t* ctx, /* Index table w } -FORCE_INLINE size_t LIZ_GetWiderMatch ( - LIZ_stream_t* ctx, +FORCE_INLINE size_t Lizard_GetWiderMatch ( + Lizard_stream_t* ctx, const BYTE* const ip, const BYTE* const iLowLimit, const BYTE* const iHighLimit, @@ -150,21 +149,21 @@ FORCE_INLINE size_t LIZ_GetWiderMatch ( size_t mlt; /* First Match */ - matchIndex = HashTable[LIZ_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)]; + matchIndex = HashTable[Lizard_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)]; - if (ctx->last_off >= LIZ_LOWESTPRICE_MIN_OFFSET) { + if (ctx->last_off >= LIZARD_LOWESTPRICE_MIN_OFFSET) { intptr_t matchIndexLO = (ip - ctx->last_off) - base; if (matchIndexLO >= lowLimit) { if (matchIndexLO >= dictLimit) { match = base + matchIndexLO; if (MEM_readMINMATCH(match) == MEM_readMINMATCH(ip)) { int back = 0; - mlt = LIZ_count(ip+MINMATCH, match+MINMATCH, iHighLimit) + MINMATCH; + mlt = Lizard_count(ip+MINMATCH, match+MINMATCH, iHighLimit) + MINMATCH; while ((ip+back > iLowLimit) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; mlt -= back; if (mlt > longest) - if ((mlt >= minMatchLongOff) || (ctx->last_off < LIZ_MAX_16BIT_OFFSET)) { + if ((mlt >= minMatchLongOff) || (ctx->last_off < LIZARD_MAX_16BIT_OFFSET)) { *matchpos = match+back; *startpos = ip+back; longest = mlt; @@ -175,12 +174,12 @@ FORCE_INLINE size_t LIZ_GetWiderMatch ( if ((U32)((dictLimit-1) - matchIndexLO) >= 3) /* intentional overflow */ if (MEM_readMINMATCH(match) == MEM_readMINMATCH(ip)) { int back=0; - mlt = LIZ_count_2segments(ip+MINMATCH, match+MINMATCH, iHighLimit, dictEnd, lowPrefixPtr) + MINMATCH; + mlt = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, iHighLimit, dictEnd, lowPrefixPtr) + MINMATCH; while ((ip+back > iLowLimit) && (matchIndexLO+back > lowLimit) && (ip[back-1] == match[back-1])) back--; mlt -= back; if (mlt > longest) - if ((mlt >= minMatchLongOff) || (ctx->last_off < LIZ_MAX_16BIT_OFFSET)) { + if ((mlt >= minMatchLongOff) || (ctx->last_off < LIZARD_MAX_16BIT_OFFSET)) { *matchpos = base + matchIndexLO + back; /* virtual matchpos */ *startpos = ip+back; longest = mlt; @@ -192,19 +191,19 @@ FORCE_INLINE size_t LIZ_GetWiderMatch ( #if MINMATCH == 3 { - U32 matchIndex3 = ctx->hashTable3[LIZ_hash3Ptr(ip, ctx->params.hashLog3)]; + U32 matchIndex3 = ctx->hashTable3[Lizard_hash3Ptr(ip, ctx->params.hashLog3)]; if (matchIndex3 < current && matchIndex3 >= lowLimit) { size_t offset = (size_t)current - matchIndex3; - if (offset < LIZ_MAX_8BIT_OFFSET) { + if (offset < LIZARD_MAX_8BIT_OFFSET) { match = ip - offset; if (match > base && MEM_readMINMATCH(ip) == MEM_readMINMATCH(match)) { - mlt = LIZ_count(ip + MINMATCH, match + MINMATCH, iHighLimit) + MINMATCH; + mlt = Lizard_count(ip + MINMATCH, match + MINMATCH, iHighLimit) + MINMATCH; int back = 0; while ((ip + back > iLowLimit) && (match + back > lowPrefixPtr) && (ip[back - 1] == match[back - 1])) back--; mlt -= back; - if (!longest || (mlt > longest && LIZ_better_price(ctx, *startpos, (*startpos - *matchpos), longest, ip, (ip - match), mlt, ctx->last_off))) { + if (!longest || (mlt > longest && Lizard_better_price(ctx, *startpos, (*startpos - *matchpos), longest, ip, (ip - match), mlt, ctx->last_off))) { *matchpos = match + back; *startpos = ip + back; longest = mlt; @@ -218,30 +217,29 @@ FORCE_INLINE size_t LIZ_GetWiderMatch ( while ((matchIndex < current) && (matchIndex >= lowLimit) && (nbAttempts)) { nbAttempts--; match = base + matchIndex; - if ((U32)(ip - match) >= LIZ_LOWESTPRICE_MIN_OFFSET) { + if ((U32)(ip - match) >= LIZARD_LOWESTPRICE_MIN_OFFSET) { if (matchIndex >= dictLimit) { if (MEM_read32(match) == MEM_read32(ip)) { int back = 0; - mlt = LIZ_count(ip+MINMATCH, match+MINMATCH, iHighLimit) + MINMATCH; + mlt = Lizard_count(ip+MINMATCH, match+MINMATCH, iHighLimit) + MINMATCH; while ((ip+back > iLowLimit) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; mlt -= back; - if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZ_MAX_16BIT_OFFSET)) - if (!longest || (mlt > longest && LIZ_better_price(ctx, *startpos, (*startpos - *matchpos), longest, ip, (ip - match), mlt, ctx->last_off))) + if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) + if (!longest || (mlt > longest && Lizard_better_price(ctx, *startpos, (*startpos - *matchpos), longest, ip, (ip - match), mlt, ctx->last_off))) { longest = mlt; *startpos = ip+back; *matchpos = match+back; } } } else { matchDict = dictBase + matchIndex; - // fprintf(stderr, "dictBase[%p]+matchIndex[%d]=match[%p] dictLimit=%d base=%p ip=%p iLimit=%p off=%d\n", dictBase, matchIndex, match, dictLimit, base, ip, iLimit, (U32)(ip-match)); if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ if (MEM_read32(matchDict) == MEM_read32(ip)) { int back=0; - mlt = LIZ_count_2segments(ip+MINMATCH, matchDict+MINMATCH, iHighLimit, dictEnd, lowPrefixPtr) + MINMATCH; + mlt = Lizard_count_2segments(ip+MINMATCH, matchDict+MINMATCH, iHighLimit, dictEnd, lowPrefixPtr) + MINMATCH; while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == matchDict[back-1])) back--; mlt -= back; - if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZ_MAX_16BIT_OFFSET)) - if (!longest || (mlt > longest && LIZ_better_price(ctx, *startpos, (*startpos - *matchpos), longest, ip, (U32)(ip - match), mlt, ctx->last_off))) + if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) + if (!longest || (mlt > longest && Lizard_better_price(ctx, *startpos, (*startpos - *matchpos), longest, ip, (U32)(ip - match), mlt, ctx->last_off))) { longest = mlt; *startpos = ip+back; *matchpos = match+back; } /* virtual matchpos */ } } @@ -255,8 +253,8 @@ FORCE_INLINE size_t LIZ_GetWiderMatch ( -FORCE_INLINE int LIZ_compress_lowestPrice( - LIZ_stream_t* const ctx, +FORCE_INLINE int Lizard_compress_lowestPrice( + Lizard_stream_t* const ctx, const BYTE* ip, const BYTE* const iend) { @@ -277,8 +275,8 @@ FORCE_INLINE int LIZ_compress_lowestPrice( /* Main Loop */ while (ip < mflimit) { - LIZ_Insert(ctx, ip); - ml = LIZ_FindMatchLowestPrice (ctx, ip, matchlimit, (&ref)); + Lizard_Insert(ctx, ip); + ml = Lizard_FindMatchLowestPrice (ctx, ip, matchlimit, (&ref)); if (!ml) { ip++; continue; } { @@ -299,8 +297,8 @@ _Search: if (ip+ml >= mflimit) { goto _Encode; } if (ml >= sufficient_len) { goto _Encode; } - LIZ_Insert(ctx, ip); - ml2 = (int)LIZ_GetWiderMatch(ctx, ip + ml - 2, anchor, matchlimit, 0, &ref2, &start2); + Lizard_Insert(ctx, ip); + ml2 = (int)Lizard_GetWiderMatch(ctx, ip + ml - 2, anchor, matchlimit, 0, &ref2, &start2); if (!ml2) goto _Encode; { @@ -310,7 +308,7 @@ _Search: // find the lowest price for encoding ml bytes best_pos = ip; - best_price = LIZ_MAX_PRICE; + best_price = LIZARD_MAX_PRICE; off0 = (int)(ip - ref); off1 = (int)(start2 - ref2); @@ -318,14 +316,14 @@ _Search: { int common0 = (int)(pos - ip); if (common0 >= MINMATCH) { - price = (int)LIZ_get_price_LZ5v2(ctx, ctx->last_off, ip, ctx->off24pos, ip - anchor, (off0 == ctx->last_off) ? 0 : off0, common0); + price = (int)Lizard_get_price_LIZv1(ctx, ctx->last_off, ip, ctx->off24pos, ip - anchor, (off0 == ctx->last_off) ? 0 : off0, common0); { int common1 = (int)(start2 + ml2 - pos); if (common1 >= MINMATCH) - price += LIZ_get_price_LZ5v2(ctx, ctx->last_off, pos, ctx->off24pos, 0, (off1 == off0) ? 0 : (off1), common1); + price += Lizard_get_price_LIZv1(ctx, ctx->last_off, pos, ctx->off24pos, 0, (off1 == off0) ? 0 : (off1), common1); else - price += LIZ_get_price_LZ5v2(ctx, ctx->last_off, pos, ctx->off24pos, common1, 0, 0); + price += Lizard_get_price_LIZv1(ctx, ctx->last_off, pos, ctx->off24pos, common1, 0, 0); } if (price < best_price) { @@ -333,19 +331,18 @@ _Search: best_pos = pos; } } else { - price = LIZ_get_price_LZ5v2(ctx, ctx->last_off, ip, ctx->off24pos, start2 - anchor, (off1 == ctx->last_off) ? 0 : off1, ml2); + price = Lizard_get_price_LIZv1(ctx, ctx->last_off, ip, ctx->off24pos, start2 - anchor, (off1 == ctx->last_off) ? 0 : off1, ml2); if (price < best_price) best_pos = pos; break; } } - // LIZ_DEBUG("%u: TRY last_off=%d literals=%u off=%u mlen=%u literals2=%u off2=%u mlen2=%u best=%d\n", (U32)(ip - ctx->inputBuffer), ctx->last_off, (U32)(ip - anchor), off0, (U32)ml, (U32)(start2 - anchor), off1, ml2, (U32)(best_pos - ip)); ml = (int)(best_pos - ip); } - if ((ml < MINMATCH) || ((ml < minMatchLongOff) && ((U32)(ip-ref) >= LIZ_MAX_16BIT_OFFSET))) + if ((ml < MINMATCH) || ((ml < minMatchLongOff) && ((U32)(ip-ref) >= LIZARD_MAX_16BIT_OFFSET))) { ip = start2; ref = ref2; @@ -356,7 +353,7 @@ _Search: _Encode: if (start0 < ip) { - if (LIZ_more_profitable(ctx, ip, (ip - ref), ml, start0, (start0 - ref0), ml0, (ref0 - ref), ctx->last_off)) + if (Lizard_more_profitable(ctx, ip, (ip - ref), ml, start0, (start0 - ref0), ml0, (ref0 - ref), ctx->last_off)) { ip = start0; ref = ref0; @@ -364,13 +361,12 @@ _Encode: } } - // if ((ml < minMatchLongOff) && ((U32)(ip-ref) >= LIZ_MAX_16BIT_OFFSET)) { printf("LIZ_encodeSequence ml=%d off=%d\n", ml, (U32)(ip-ref)); exit(0); } - if (LIZ_encodeSequence_LZ5v2(ctx, &ip, &anchor, ml, ((ip - ref == ctx->last_off) ? ip : ref))) return 0; + if (Lizard_encodeSequence_LIZv1(ctx, &ip, &anchor, ml, ((ip - ref == ctx->last_off) ? ip : ref))) return 0; } /* Encode Last Literals */ ip = iend; - if (LIZ_encodeLastLiterals_LZ5v2(ctx, &ip, &anchor)) goto _output_error; + if (Lizard_encodeLastLiterals_LIZv1(ctx, &ip, &anchor)) goto _output_error; /* End */ return 1; diff --git a/C/lizard/liz_parser_nochain.h b/C/lizard/lizard_parser_nochain.h similarity index 72% rename from C/lizard/liz_parser_nochain.h rename to C/lizard/lizard_parser_nochain.h index 034828ff..e8abb586 100644 --- a/C/lizard/liz_parser_nochain.h +++ b/C/lizard/lizard_parser_nochain.h @@ -1,11 +1,11 @@ #define OPTIMAL_ML (int)((ML_MASK_LZ4-1)+MINMATCH) -//#define LIZ_NOCHAIN_HASH_FUNCTION(ip, hashLog) LIZ_hashPtr(ip, hashLog, ctx->params.searchLength) -#define LIZ_NOCHAIN_HASH_FUNCTION(ip, hashLog) LIZ_hash5Ptr(ip, hashLog) -#define LIZ_NOCHAIN_MIN_OFFSET 8 +//#define LIZARD_NOCHAIN_HASH_FUNCTION(ip, hashLog) Lizard_hashPtr(ip, hashLog, ctx->params.searchLength) +#define LIZARD_NOCHAIN_HASH_FUNCTION(ip, hashLog) Lizard_hash5Ptr(ip, hashLog) +#define LIZARD_NOCHAIN_MIN_OFFSET 8 /* Update chains up to ip (excluded) */ -FORCE_INLINE void LIZ_InsertNoChain (LIZ_stream_t* ctx, const BYTE* ip) +FORCE_INLINE void Lizard_InsertNoChain (Lizard_stream_t* ctx, const BYTE* ip) { U32* const hashTable = ctx->hashTable; const BYTE* const base = ctx->base; @@ -14,8 +14,8 @@ FORCE_INLINE void LIZ_InsertNoChain (LIZ_stream_t* ctx, const BYTE* ip) const int hashLog = ctx->params.hashLog; while (idx < target) { - size_t const h = LIZ_NOCHAIN_HASH_FUNCTION(base+idx, hashLog); - if (idx >= hashTable[h] + LIZ_NOCHAIN_MIN_OFFSET) + size_t const h = LIZARD_NOCHAIN_HASH_FUNCTION(base+idx, hashLog); + if ((hashTable[h] >= idx) || (idx >= hashTable[h] + LIZARD_NOCHAIN_MIN_OFFSET)) hashTable[h] = idx; idx++; } @@ -24,8 +24,7 @@ FORCE_INLINE void LIZ_InsertNoChain (LIZ_stream_t* ctx, const BYTE* ip) } - -FORCE_INLINE int LIZ_InsertAndFindBestMatchNoChain (LIZ_stream_t* ctx, /* Index table will be updated */ +FORCE_INLINE int Lizard_InsertAndFindBestMatchNoChain (Lizard_stream_t* ctx, /* Index table will be updated */ const BYTE* ip, const BYTE* const iLimit, const BYTE** matchpos) { @@ -40,32 +39,32 @@ FORCE_INLINE int LIZ_InsertAndFindBestMatchNoChain (LIZ_stream_t* ctx, /* Inde size_t ml=0; const int hashLog = ctx->params.hashLog; const U32 maxDistance = (1 << ctx->params.windowLog) - 1; - const U32 lowLimit = (ctx->lowLimit + maxDistance >= (U32)(ip-base)) ? ctx->lowLimit : (U32)(ip - base) - maxDistance; + const U32 current = (U32)(ip - base); + const U32 lowLimit = (ctx->lowLimit + maxDistance >= current) ? ctx->lowLimit : current - maxDistance; /* HC4 match finder */ - LIZ_InsertNoChain(ctx, ip); - matchIndex = HashTable[LIZ_NOCHAIN_HASH_FUNCTION(ip, hashLog)]; + Lizard_InsertNoChain(ctx, ip); + matchIndex = HashTable[LIZARD_NOCHAIN_HASH_FUNCTION(ip, hashLog)]; - if (matchIndex >= lowLimit) { + if ((matchIndex < current) && (matchIndex >= lowLimit)) { if (matchIndex >= dictLimit) { match = base + matchIndex; -#if LIZ_NOCHAIN_MIN_OFFSET > 0 - if ((U32)(ip - match) >= LIZ_NOCHAIN_MIN_OFFSET) +#if LIZARD_NOCHAIN_MIN_OFFSET > 0 + if ((U32)(ip - match) >= LIZARD_NOCHAIN_MIN_OFFSET) #endif if (*(match+ml) == *(ip+ml) && (MEM_read32(match) == MEM_read32(ip))) { - size_t const mlt = LIZ_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; + size_t const mlt = Lizard_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; if (mlt > ml) { ml = mlt; *matchpos = match; } } } else { match = dictBase + matchIndex; -// fprintf(stderr, "dictBase[%p]+matchIndex[%d]=match[%p] dictLimit=%d base=%p ip=%p iLimit=%p off=%d\n", dictBase, matchIndex, match, dictLimit, base, ip, iLimit, (U32)(ip-match)); -#if LIZ_NOCHAIN_MIN_OFFSET > 0 - if ((U32)(ip - (base + matchIndex)) >= LIZ_NOCHAIN_MIN_OFFSET) +#if LIZARD_NOCHAIN_MIN_OFFSET > 0 + if ((U32)(ip - (base + matchIndex)) >= LIZARD_NOCHAIN_MIN_OFFSET) #endif if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ if (MEM_read32(match) == MEM_read32(ip)) { - size_t mlt = LIZ_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH; + size_t mlt = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH; if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */ } } @@ -75,8 +74,8 @@ FORCE_INLINE int LIZ_InsertAndFindBestMatchNoChain (LIZ_stream_t* ctx, /* Inde } -FORCE_INLINE int LIZ_InsertAndGetWiderMatchNoChain ( - LIZ_stream_t* ctx, +FORCE_INLINE int Lizard_InsertAndGetWiderMatchNoChain ( + Lizard_stream_t* ctx, const BYTE* const ip, const BYTE* const iLowLimit, const BYTE* const iHighLimit, @@ -94,21 +93,22 @@ FORCE_INLINE int LIZ_InsertAndGetWiderMatchNoChain ( int LLdelta = (int)(ip-iLowLimit); const int hashLog = ctx->params.hashLog; const U32 maxDistance = (1 << ctx->params.windowLog) - 1; - const U32 lowLimit = (ctx->lowLimit + maxDistance >= (U32)(ip-base)) ? ctx->lowLimit : (U32)(ip - base) - maxDistance; - - /* First Match */ - LIZ_InsertNoChain(ctx, ip); - matchIndex = HashTable[LIZ_NOCHAIN_HASH_FUNCTION(ip, hashLog)]; + const U32 current = (U32)(ip - base); + const U32 lowLimit = (ctx->lowLimit + maxDistance >= current) ? ctx->lowLimit : current - maxDistance; - if (matchIndex>=lowLimit) { + /* First Match */ + Lizard_InsertNoChain(ctx, ip); + matchIndex = HashTable[LIZARD_NOCHAIN_HASH_FUNCTION(ip, hashLog)]; + + if ((matchIndex < current) && (matchIndex >= lowLimit)) { if (matchIndex >= dictLimit) { const BYTE* match = base + matchIndex; -#if LIZ_NOCHAIN_MIN_OFFSET > 0 - if ((U32)(ip - match) >= LIZ_NOCHAIN_MIN_OFFSET) +#if LIZARD_NOCHAIN_MIN_OFFSET > 0 + if ((U32)(ip - match) >= LIZARD_NOCHAIN_MIN_OFFSET) #endif if (*(iLowLimit + longest) == *(match - LLdelta + longest)) { if (MEM_read32(match) == MEM_read32(ip)) { - int mlt = MINMATCH + LIZ_count(ip+MINMATCH, match+MINMATCH, iHighLimit); + int mlt = MINMATCH + Lizard_count(ip+MINMATCH, match+MINMATCH, iHighLimit); int back = 0; while ((ip+back > iLowLimit) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; mlt -= back; @@ -122,13 +122,13 @@ FORCE_INLINE int LIZ_InsertAndGetWiderMatchNoChain ( } } else { const BYTE* match = dictBase + matchIndex; -#if LIZ_NOCHAIN_MIN_OFFSET > 0 - if ((U32)(ip - (base + matchIndex)) >= LIZ_NOCHAIN_MIN_OFFSET) +#if LIZARD_NOCHAIN_MIN_OFFSET > 0 + if ((U32)(ip - (base + matchIndex)) >= LIZARD_NOCHAIN_MIN_OFFSET) #endif if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ if (MEM_read32(match) == MEM_read32(ip)) { int back=0; - size_t mlt = LIZ_count_2segments(ip+MINMATCH, match+MINMATCH, iHighLimit, dictEnd, lowPrefixPtr) + MINMATCH; + size_t mlt = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, iHighLimit, dictEnd, lowPrefixPtr) + MINMATCH; while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == match[back-1])) back--; mlt -= back; if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; } @@ -140,8 +140,8 @@ FORCE_INLINE int LIZ_InsertAndGetWiderMatchNoChain ( } -FORCE_INLINE int LIZ_compress_noChain ( - LIZ_stream_t* const ctx, +FORCE_INLINE int Lizard_compress_noChain ( + Lizard_stream_t* const ctx, const BYTE* ip, const BYTE* const iend) { @@ -163,7 +163,7 @@ FORCE_INLINE int LIZ_compress_noChain ( /* Main Loop */ while (ip < mflimit) { - ml = LIZ_InsertAndFindBestMatchNoChain (ctx, ip, matchlimit, (&ref)); + ml = Lizard_InsertAndFindBestMatchNoChain (ctx, ip, matchlimit, (&ref)); if (!ml) { ip++; continue; } /* saved, in case we would skip too much */ @@ -173,11 +173,11 @@ FORCE_INLINE int LIZ_compress_noChain ( _Search2: if (ip+ml < mflimit) - ml2 = LIZ_InsertAndGetWiderMatchNoChain(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2); + ml2 = Lizard_InsertAndGetWiderMatchNoChain(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2); else ml2 = ml; if (ml2 == ml) { /* No better match */ - if (LIZ_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; continue; } @@ -218,16 +218,16 @@ _Search3: /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */ if (start2 + ml2 < mflimit) - ml3 = LIZ_InsertAndGetWiderMatchNoChain(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3); + ml3 = Lizard_InsertAndGetWiderMatchNoChain(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3); else ml3 = ml2; if (ml3 == ml2) { /* No better match : 2 sequences to encode */ /* ip & ref are known; Now for ml */ if (start2 < ip+ml) ml = (int)(start2 - ip); /* Now, encode 2 sequences */ - if (LIZ_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; ip = start2; - if (LIZ_encodeSequence_LZ4(ctx, &ip, &anchor, ml2, ref2)) return 0; + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml2, ref2)) return 0; continue; } @@ -245,7 +245,7 @@ _Search3: } } - if (LIZ_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; ip = start3; ref = ref3; ml = ml3; @@ -273,7 +273,7 @@ _Search3: if (ip + ml > start2 + ml2 - MINMATCH) { ml = (int)(start2 - ip) + ml2 - MINMATCH; if (ml < MINMATCH) { // match2 doesn't fit, remove it - if (LIZ_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; ip = start3; ref = ref3; ml = ml3; @@ -294,7 +294,7 @@ _Search3: ml = (int)(start2 - ip); } } - if (LIZ_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; ip = start2; ref = ref2; @@ -309,7 +309,7 @@ _Search3: /* Encode Last Literals */ ip = iend; - if (LIZ_encodeLastLiterals_LZ4(ctx, &ip, &anchor)) goto _output_error; + if (Lizard_encodeLastLiterals_LZ4(ctx, &ip, &anchor)) goto _output_error; /* End */ return 1; diff --git a/C/lizard/liz_parser_optimal.h b/C/lizard/lizard_parser_optimal.h similarity index 63% rename from C/lizard/liz_parser_optimal.h rename to C/lizard/lizard_parser_optimal.h index b9c00752..8c9b83e9 100644 --- a/C/lizard/liz_parser_optimal.h +++ b/C/lizard/lizard_parser_optimal.h @@ -1,18 +1,18 @@ -#define LIZ_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__) -#define LIZ_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__) -#define LIZ_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__) +#define LIZARD_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__) +#define LIZARD_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__) +#define LIZARD_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__) -#define LIZ_OPTIMAL_MIN_OFFSET 8 -#define LIZ_OPT_NUM (1<<12) +#define LIZARD_OPTIMAL_MIN_OFFSET 8 +#define LIZARD_OPT_NUM (1<<12) #define REPMINMATCH 1 -FORCE_INLINE size_t LIZ_get_price(LIZ_stream_t* const ctx, int rep, const BYTE *ip, const BYTE *off24pos, size_t litLength, U32 offset, size_t matchLength) +FORCE_INLINE size_t Lizard_get_price(Lizard_stream_t* const ctx, int rep, const BYTE *ip, const BYTE *off24pos, size_t litLength, U32 offset, size_t matchLength) { - if (ctx->params.decompressType == LIZ_coderwords_LZ4) - return LIZ_get_price_LZ4(ctx, ip, litLength, offset, matchLength); + if (ctx->params.decompressType == Lizard_coderwords_LZ4) + return Lizard_get_price_LZ4(ctx, ip, litLength, offset, matchLength); - return LIZ_get_price_LZ5v2(ctx, rep, ip, off24pos, litLength, offset, matchLength); + return Lizard_get_price_LIZv1(ctx, rep, ip, off24pos, litLength, offset, matchLength); } @@ -22,7 +22,7 @@ typedef struct int off; int len; int back; -} LIZ_match_t; +} Lizard_match_t; typedef struct { @@ -32,11 +32,11 @@ typedef struct int litlen; int rep; const BYTE* off24pos; -} LIZ_optimal_t; +} Lizard_optimal_t; /* Update chains up to ip (excluded) */ -FORCE_INLINE void LIZ_BinTree_Insert(LIZ_stream_t* ctx, const BYTE* ip) +FORCE_INLINE void Lizard_BinTree_Insert(Lizard_stream_t* ctx, const BYTE* ip) { #if MINMATCH == 3 U32* HashTable3 = ctx->hashTable3; @@ -45,7 +45,7 @@ FORCE_INLINE void LIZ_BinTree_Insert(LIZ_stream_t* ctx, const BYTE* ip) U32 idx = ctx->nextToUpdate; while(idx < target) { - HashTable3[LIZ_hash3Ptr(base+idx, ctx->params.hashLog3)] = idx; + HashTable3[Lizard_hash3Ptr(base+idx, ctx->params.hashLog3)] = idx; idx++; } @@ -57,13 +57,13 @@ FORCE_INLINE void LIZ_BinTree_Insert(LIZ_stream_t* ctx, const BYTE* ip) -FORCE_INLINE int LIZ_GetAllMatches ( - LIZ_stream_t* ctx, +FORCE_INLINE int Lizard_GetAllMatches ( + Lizard_stream_t* ctx, const BYTE* const ip, const BYTE* const iLowLimit, const BYTE* const iHighLimit, size_t best_mlen, - LIZ_match_t* matches) + Lizard_match_t* matches) { U32* const chainTable = ctx->chainTable; U32* const HashTable = ctx->hashTable; @@ -88,19 +88,19 @@ FORCE_INLINE int LIZ_GetAllMatches ( if (ip + MINMATCH > iHighLimit) return 0; /* First Match */ - HashPos = &HashTable[LIZ_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)]; + HashPos = &HashTable[Lizard_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)]; matchIndex = *HashPos; #if MINMATCH == 3 { U32* const HashTable3 = ctx->hashTable3; - U32* HashPos3 = &HashTable3[LIZ_hash3Ptr(ip, ctx->params.hashLog3)]; + U32* HashPos3 = &HashTable3[Lizard_hash3Ptr(ip, ctx->params.hashLog3)]; if ((*HashPos3 < current) && (*HashPos3 >= lowLimit)) { size_t offset = current - *HashPos3; - if (offset < LIZ_MAX_8BIT_OFFSET) { + if (offset < LIZARD_MAX_8BIT_OFFSET) { match = ip - offset; if (match > base && MEM_readMINMATCH(ip) == MEM_readMINMATCH(match)) { - size_t mlt = LIZ_count(ip + MINMATCH, match + MINMATCH, iHighLimit) + MINMATCH; + size_t mlt = Lizard_count(ip + MINMATCH, match + MINMATCH, iHighLimit) + MINMATCH; int back = 0; while ((ip + back > iLowLimit) && (match + back > lowPrefixPtr) && (ip[back - 1] == match[back - 1])) back--; @@ -127,15 +127,15 @@ FORCE_INLINE int LIZ_GetAllMatches ( while ((matchIndex < current) && (matchIndex >= lowLimit) && (nbAttempts)) { nbAttempts--; match = base + matchIndex; - if ((U32)(ip - match) >= LIZ_OPTIMAL_MIN_OFFSET) { + if ((U32)(ip - match) >= LIZARD_OPTIMAL_MIN_OFFSET) { if (matchIndex >= dictLimit) { if ((/*fullSearch ||*/ ip[best_mlen] == match[best_mlen]) && (MEM_readMINMATCH(match) == MEM_readMINMATCH(ip))) { int back = 0; - mlt = LIZ_count(ip+MINMATCH, match+MINMATCH, iHighLimit) + MINMATCH; + mlt = Lizard_count(ip+MINMATCH, match+MINMATCH, iHighLimit) + MINMATCH; while ((ip+back > iLowLimit) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; mlt -= back; - if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZ_MAX_16BIT_OFFSET)) + if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) if (mlt > best_mlen) { best_mlen = mlt; matches[mnum].off = (int)(ip - match); @@ -143,7 +143,7 @@ FORCE_INLINE int LIZ_GetAllMatches ( matches[mnum].back = -back; mnum++; - if (best_mlen > LIZ_OPT_NUM) break; + if (best_mlen > LIZARD_OPT_NUM) break; } } } else { @@ -152,11 +152,11 @@ FORCE_INLINE int LIZ_GetAllMatches ( if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ if (MEM_readMINMATCH(matchDict) == MEM_readMINMATCH(ip)) { int back=0; - mlt = LIZ_count_2segments(ip+MINMATCH, matchDict+MINMATCH, iHighLimit, dictEnd, lowPrefixPtr) + MINMATCH; + mlt = Lizard_count_2segments(ip+MINMATCH, matchDict+MINMATCH, iHighLimit, dictEnd, lowPrefixPtr) + MINMATCH; while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == matchDict[back-1])) back--; mlt -= back; - if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZ_MAX_16BIT_OFFSET)) + if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) if (mlt > best_mlen) { best_mlen = mlt; matches[mnum].off = (int)(ip - match); @@ -164,7 +164,7 @@ FORCE_INLINE int LIZ_GetAllMatches ( matches[mnum].back = -back; mnum++; - if (best_mlen > LIZ_OPT_NUM) break; + if (best_mlen > LIZARD_OPT_NUM) break; } } } @@ -178,12 +178,12 @@ FORCE_INLINE int LIZ_GetAllMatches ( -FORCE_INLINE int LIZ_BinTree_GetAllMatches ( - LIZ_stream_t* ctx, +FORCE_INLINE int Lizard_BinTree_GetAllMatches ( + Lizard_stream_t* ctx, const BYTE* const ip, const BYTE* const iHighLimit, size_t best_mlen, - LIZ_match_t* matches) + Lizard_match_t* matches) { U32* const chainTable = ctx->chainTable; U32* const HashTable = ctx->hashTable; @@ -208,21 +208,21 @@ FORCE_INLINE int LIZ_BinTree_GetAllMatches ( if (ip + MINMATCH > iHighLimit) return 0; /* First Match */ - HashPos = &HashTable[LIZ_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)]; + HashPos = &HashTable[Lizard_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)]; matchIndex = *HashPos; #if MINMATCH == 3 { - U32* HashPos3 = &ctx->hashTable3[LIZ_hash3Ptr(ip, ctx->params.hashLog3)]; + U32* HashPos3 = &ctx->hashTable3[Lizard_hash3Ptr(ip, ctx->params.hashLog3)]; if ((*HashPos3 < current) && (*HashPos3 >= lowLimit)) { size_t offset = current - *HashPos3; - if (offset < LIZ_MAX_8BIT_OFFSET) { + if (offset < LIZARD_MAX_8BIT_OFFSET) { match = ip - offset; if (match > base && MEM_readMINMATCH(ip) == MEM_readMINMATCH(match)) { - mlt = LIZ_count(ip + MINMATCH, match + MINMATCH, iHighLimit) + MINMATCH; + mlt = Lizard_count(ip + MINMATCH, match + MINMATCH, iHighLimit) + MINMATCH; matches[mnum].off = (int)offset; matches[mnum].len = (int)mlt; @@ -250,16 +250,16 @@ FORCE_INLINE int LIZ_BinTree_GetAllMatches ( if (matchIndex >= dictLimit) { match = base + matchIndex; // if (ip[mlt] == match[mlt]) - mlt = LIZ_count(ip, match, iHighLimit); + mlt = Lizard_count(ip, match, iHighLimit); } else { match = dictBase + matchIndex; - mlt = LIZ_count_2segments(ip, match, iHighLimit, dictEnd, lowPrefixPtr); + mlt = Lizard_count_2segments(ip, match, iHighLimit, dictEnd, lowPrefixPtr); if (matchIndex + (int)mlt >= dictLimit) match = base + matchIndex; /* to prepare for next usage of match[mlt] */ } - if ((U32)(current - matchIndex) >= LIZ_OPTIMAL_MIN_OFFSET) { - if ((mlt >= minMatchLongOff) || ((U32)(current - matchIndex) < LIZ_MAX_16BIT_OFFSET)) + if ((U32)(current - matchIndex) >= LIZARD_OPTIMAL_MIN_OFFSET) { + if ((mlt >= minMatchLongOff) || ((U32)(current - matchIndex) < LIZARD_MAX_16BIT_OFFSET)) if (mlt > best_mlen) { best_mlen = mlt; matches[mnum].off = (int)(current - matchIndex); @@ -267,7 +267,7 @@ FORCE_INLINE int LIZ_BinTree_GetAllMatches ( matches[mnum].back = 0; mnum++; - if (mlt > LIZ_OPT_NUM) break; + if (mlt > LIZARD_OPT_NUM) break; if (ip + mlt >= iHighLimit) break; } } else { @@ -276,9 +276,9 @@ FORCE_INLINE int LIZ_BinTree_GetAllMatches ( size_t newml = 0, newoff = 0; do { newoff += (int)(current - matchIndex); - } while (newoff < LIZ_OPTIMAL_MIN_OFFSET); + } while (newoff < LIZARD_OPTIMAL_MIN_OFFSET); newMatchIndex = current - newoff; - if (newMatchIndex >= dictLimit) newml = LIZ_count(ip, base + newMatchIndex, iHighLimit); + if (newMatchIndex >= dictLimit) newml = Lizard_count(ip, base + newMatchIndex, iHighLimit); // printf("%d: off=%d mlt=%d\n", (U32)current, (U32)(current - matchIndex), (int)mlt); // printf("%d: newoff=%d newml=%d\n", (U32)current, (int)newoff, (int)newml); @@ -290,7 +290,7 @@ FORCE_INLINE int LIZ_BinTree_GetAllMatches ( matches[mnum].back = 0; mnum++; - if (newml > LIZ_OPT_NUM) break; + if (newml > LIZARD_OPT_NUM) break; if (ip + newml >= iHighLimit) break; } #endif @@ -322,22 +322,22 @@ FORCE_INLINE int LIZ_BinTree_GetAllMatches ( #define SET_PRICE(pos, mlen, offset, litlen, price) \ { \ - while (last_pos < pos) { opt[last_pos+1].price = LIZ_MAX_PRICE; last_pos++; } \ + while (last_pos < pos) { opt[last_pos+1].price = LIZARD_MAX_PRICE; last_pos++; } \ opt[pos].mlen = (int)mlen; \ opt[pos].off = (int)offset; \ opt[pos].litlen = (int)litlen; \ opt[pos].price = (int)price; \ - LIZ_LOG_PARSER("%d: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-source), pos, last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \ + LIZARD_LOG_PARSER("%d: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-source), pos, last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \ } -FORCE_INLINE int LIZ_compress_optimalPrice( - LIZ_stream_t* const ctx, +FORCE_INLINE int Lizard_compress_optimalPrice( + Lizard_stream_t* const ctx, const BYTE* ip, const BYTE* const iend) { - LIZ_optimal_t opt[LIZ_OPT_NUM + 4]; - LIZ_match_t matches[LIZ_OPT_NUM + 1]; + Lizard_optimal_t opt[LIZARD_OPT_NUM + 4]; + Lizard_match_t matches[LIZARD_OPT_NUM + 1]; const BYTE *inr; size_t res, cur, cur2, skip_num = 0; size_t i, llen, litlen, mlen, best_mlen, price, offset, best_off, match_num, last_pos; @@ -356,29 +356,29 @@ FORCE_INLINE int LIZ_compress_optimalPrice( const size_t sufficient_len = ctx->params.sufficientLength; const int faster_get_matches = (ctx->params.fullSearch == 0); const size_t minMatchLongOff = ctx->params.minMatchLongOff; - const int lz5OptimalMinOffset = (ctx->params.decompressType == LIZ_coderwords_LZ4) ? (1<<30) : LIZ_OPTIMAL_MIN_OFFSET; - const size_t repMinMatch = (ctx->params.decompressType == LIZ_coderwords_LZ4) ? MINMATCH : REPMINMATCH; + const int lizardOptimalMinOffset = (ctx->params.decompressType == Lizard_coderwords_LZ4) ? (1<<30) : LIZARD_OPTIMAL_MIN_OFFSET; + const size_t repMinMatch = (ctx->params.decompressType == Lizard_coderwords_LZ4) ? MINMATCH : REPMINMATCH; /* Main Loop */ while (ip < mflimit) { - memset(opt, 0, sizeof(LIZ_optimal_t)); + memset(opt, 0, sizeof(Lizard_optimal_t)); last_pos = 0; llen = ip - anchor; /* check rep code */ - if (ctx->last_off >= lz5OptimalMinOffset) { + if (ctx->last_off >= lizardOptimalMinOffset) { intptr_t matchIndexLO = (ip - ctx->last_off) - base; mlen = 0; if ((matchIndexLO >= lowLimit) && (base + matchIndexLO + maxDistance >= ip)) { if (matchIndexLO >= dictLimit) { - mlen = LIZ_count(ip, base + matchIndexLO, matchlimit); + mlen = Lizard_count(ip, base + matchIndexLO, matchlimit); } else { - mlen = LIZ_count_2segments(ip, dictBase + matchIndexLO, matchlimit, dictEnd, lowPrefixPtr); + mlen = Lizard_count_2segments(ip, dictBase + matchIndexLO, matchlimit, dictEnd, lowPrefixPtr); } } if (mlen >= REPMINMATCH) { - if (mlen > sufficient_len || mlen >= LIZ_OPT_NUM) { + if (mlen > sufficient_len || mlen >= LIZARD_OPT_NUM) { best_mlen = mlen; best_off = 0; cur = 0; last_pos = 1; goto encode; } @@ -386,7 +386,7 @@ FORCE_INLINE int LIZ_compress_optimalPrice( do { litlen = 0; - price = LIZ_get_price(ctx, ctx->last_off, ip, ctx->off24pos, llen, 0, mlen); + price = Lizard_get_price(ctx, ctx->last_off, ip, ctx->off24pos, llen, 0, mlen); if (mlen > last_pos || price < (size_t)opt[mlen].price) SET_PRICE(mlen, mlen, 0, litlen, price); mlen--; @@ -399,16 +399,16 @@ FORCE_INLINE int LIZ_compress_optimalPrice( match_num = 0; else { - if (ctx->params.parserType == LIZ_parser_optimalPrice) { - LIZ_Insert(ctx, ip); - match_num = LIZ_GetAllMatches(ctx, ip, ip, matchlimit, last_pos, matches); + if (ctx->params.parserType == Lizard_parser_optimalPrice) { + Lizard_Insert(ctx, ip); + match_num = Lizard_GetAllMatches(ctx, ip, ip, matchlimit, last_pos, matches); } else { - LIZ_BinTree_Insert(ctx, ip); - match_num = LIZ_BinTree_GetAllMatches(ctx, ip, matchlimit, last_pos, matches); + Lizard_BinTree_Insert(ctx, ip); + match_num = Lizard_BinTree_GetAllMatches(ctx, ip, matchlimit, last_pos, matches); } } - LIZ_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-source), match_num, last_pos); + LIZARD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-source), match_num, last_pos); if (!last_pos && !match_num) { ip++; continue; } if (match_num && (size_t)matches[match_num-1].len > sufficient_len) { @@ -424,13 +424,13 @@ FORCE_INLINE int LIZ_compress_optimalPrice( for (i = 0; i < match_num; i++) { mlen = (i>0) ? (size_t)matches[i-1].len+1 : best_mlen; - best_mlen = (matches[i].len < LIZ_OPT_NUM) ? matches[i].len : LIZ_OPT_NUM; - LIZ_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-source), matches[i].len, matches[i].off, best_mlen, last_pos); + best_mlen = (matches[i].len < LIZARD_OPT_NUM) ? matches[i].len : LIZARD_OPT_NUM; + LIZARD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-source), matches[i].len, matches[i].off, best_mlen, last_pos); while (mlen <= best_mlen){ litlen = 0; - price = LIZ_get_price(ctx, ctx->last_off, ip, ctx->off24pos, llen + litlen, matches[i].off, mlen); + price = Lizard_get_price(ctx, ctx->last_off, ip, ctx->off24pos, llen + litlen, matches[i].off, mlen); - if ((mlen >= minMatchLongOff) || (matches[i].off < LIZ_MAX_16BIT_OFFSET)) + if ((mlen >= minMatchLongOff) || (matches[i].off < LIZARD_MAX_16BIT_OFFSET)) if (mlen > last_pos || price < (size_t)opt[mlen].price) SET_PRICE(mlen, mlen, matches[i].off, litlen, price); mlen++; @@ -453,21 +453,21 @@ FORCE_INLINE int LIZ_compress_optimalPrice( litlen = opt[cur-1].litlen + 1; if (cur != litlen) { - price = opt[cur - litlen].price + LIZ_get_price(ctx, opt[cur-litlen].rep, inr, ctx->off24pos, litlen, 0, 0); - LIZ_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-source), cur - litlen, opt[cur - litlen].price, price, cur, litlen); + price = opt[cur - litlen].price + Lizard_get_price(ctx, opt[cur-litlen].rep, inr, ctx->off24pos, litlen, 0, 0); + LIZARD_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-source), cur - litlen, opt[cur - litlen].price, price, cur, litlen); } else { - price = LIZ_get_price(ctx, ctx->last_off, inr, ctx->off24pos, llen + litlen, 0, 0); - LIZ_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-source), price, cur, litlen, llen); + price = Lizard_get_price(ctx, ctx->last_off, inr, ctx->off24pos, llen + litlen, 0, 0); + LIZARD_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-source), price, cur, litlen, llen); } } else { litlen = 1; - price = opt[cur - 1].price + LIZ_get_price(ctx, opt[cur-1].rep, inr, ctx->off24pos, litlen, 0, 0); - LIZ_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-source), price, cur, litlen, LIZ_get_price(ctx, rep, inr, ctx->off24pos, litlen, 0, 0)); + price = opt[cur - 1].price + Lizard_get_price(ctx, opt[cur-1].rep, inr, ctx->off24pos, litlen, 0, 0); + LIZARD_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-source), price, cur, litlen, Lizard_get_price(ctx, rep, inr, ctx->off24pos, litlen, 0, 0)); } mlen = 1; best_mlen = 0; - LIZ_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-source), price, cur, opt[cur].price); + LIZARD_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-source), price, cur, opt[cur].price); if (cur > last_pos || price <= (size_t)opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) SET_PRICE(cur, mlen, -1, litlen, price); @@ -483,11 +483,11 @@ FORCE_INLINE int LIZ_compress_optimalPrice( if (offset < 1) { opt[cur].rep = opt[cur-mlen].rep; opt[cur].off24pos = opt[cur-mlen].off24pos; - LIZ_LOG_PARSER("%d: COPYREP1 cur=%d mlen=%d rep=%d\n", (int)(inr-source), cur, mlen, opt[cur-mlen].rep); + LIZARD_LOG_PARSER("%d: COPYREP1 cur=%d mlen=%d rep=%d\n", (int)(inr-source), cur, mlen, opt[cur-mlen].rep); } else { opt[cur].rep = (int)offset; - opt[cur].off24pos = (offset >= LIZ_MAX_16BIT_OFFSET) ? inr : opt[cur-mlen].off24pos; - LIZ_LOG_PARSER("%d: COPYREP2 cur=%d offset=%d rep=%d\n", (int)(inr-source), cur, offset, opt[cur].rep); + opt[cur].off24pos = (offset >= LIZARD_MAX_16BIT_OFFSET) ? inr : opt[cur-mlen].off24pos; + LIZARD_LOG_PARSER("%d: COPYREP2 cur=%d offset=%d rep=%d\n", (int)(inr-source), cur, offset, opt[cur].rep); } } else { opt[cur].rep = opt[cur-1].rep; // copy rep @@ -495,28 +495,28 @@ FORCE_INLINE int LIZ_compress_optimalPrice( } rep = opt[cur].rep; - LIZ_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep); + LIZARD_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep); /* check rep code */ - if (opt[cur].rep >= lz5OptimalMinOffset) { + if (opt[cur].rep >= lizardOptimalMinOffset) { intptr_t matchIndexLO = (inr - opt[cur].rep) - base; mlen = 0; if ((matchIndexLO >= lowLimit) && (base + matchIndexLO + maxDistance >= inr)) { if (matchIndexLO >= dictLimit) { - mlen = LIZ_count(inr, base + matchIndexLO, matchlimit); + mlen = Lizard_count(inr, base + matchIndexLO, matchlimit); } else { - mlen = LIZ_count_2segments(inr, dictBase + matchIndexLO, matchlimit, dictEnd, lowPrefixPtr); + mlen = Lizard_count_2segments(inr, dictBase + matchIndexLO, matchlimit, dictEnd, lowPrefixPtr); } } if (mlen >= REPMINMATCH/* && mlen > best_mlen*/) { - LIZ_LOG_PARSER("%d: try REP rep=%d mlen=%d\n", (int)(inr-source), opt[cur].rep, mlen); - LIZ_LOG_PARSER("%d: Found REP mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-source), mlen, 0, opt[cur].rep, cur, opt[cur].off); + LIZARD_LOG_PARSER("%d: try REP rep=%d mlen=%d\n", (int)(inr-source), opt[cur].rep, mlen); + LIZARD_LOG_PARSER("%d: Found REP mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-source), mlen, 0, opt[cur].rep, cur, opt[cur].off); - if (mlen > sufficient_len || cur + mlen >= LIZ_OPT_NUM) { + if (mlen > sufficient_len || cur + mlen >= LIZARD_OPT_NUM) { best_mlen = mlen; best_off = 0; - LIZ_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-source), sufficient_len, best_mlen, best_off, last_pos); + LIZARD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-source), sufficient_len, best_mlen, best_off, last_pos); last_pos = cur + 1; goto encode; } @@ -532,19 +532,19 @@ FORCE_INLINE int LIZ_compress_optimalPrice( litlen = opt[cur].litlen; if (cur != litlen) { - price = opt[cur - litlen].price + LIZ_get_price(ctx, rep, inr, opt[cur].off24pos, litlen, 0, mlen); - LIZ_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-source), cur - litlen, opt[cur - litlen].price, price, cur, litlen); + price = opt[cur - litlen].price + Lizard_get_price(ctx, rep, inr, opt[cur].off24pos, litlen, 0, mlen); + LIZARD_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-source), cur - litlen, opt[cur - litlen].price, price, cur, litlen); } else { - price = LIZ_get_price(ctx, rep, inr, ctx->off24pos, llen + litlen, 0, mlen); - LIZ_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-source), price, cur, litlen, llen); + price = Lizard_get_price(ctx, rep, inr, ctx->off24pos, llen + litlen, 0, mlen); + LIZARD_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-source), price, cur, litlen, llen); } } else { litlen = 0; - price = opt[cur].price + LIZ_get_price(ctx, rep, inr, opt[cur].off24pos, litlen, 0, mlen); - LIZ_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d getprice=%d\n", (int)(inr-source), price, cur, litlen, LIZ_get_price(ctx, rep, inr, opt[cur].off24pos, litlen, 0, mlen - MINMATCH)); + price = opt[cur].price + Lizard_get_price(ctx, rep, inr, opt[cur].off24pos, litlen, 0, mlen); + LIZARD_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d getprice=%d\n", (int)(inr-source), price, cur, litlen, Lizard_get_price(ctx, rep, inr, opt[cur].off24pos, litlen, 0, mlen - MINMATCH)); } - LIZ_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), mlen, 0, price, litlen, cur - litlen, opt[cur - litlen].price); + LIZARD_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), mlen, 0, price, litlen, cur - litlen, opt[cur - litlen].price); if (cur + mlen > last_pos || price <= (size_t)opt[cur + mlen].price) // || ((price == opt[cur + mlen].price) && (opt[cur].mlen == 1) && (cur != litlen))) // at equal price prefer REP instead of MATCH SET_PRICE(cur + mlen, mlen, 0, litlen, price); @@ -559,14 +559,14 @@ FORCE_INLINE int LIZ_compress_optimalPrice( continue; } - if (ctx->params.parserType == LIZ_parser_optimalPrice) { - LIZ_Insert(ctx, inr); - match_num = LIZ_GetAllMatches(ctx, inr, ip, matchlimit, best_mlen, matches); - LIZ_LOG_PARSER("%d: LIZ_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); + if (ctx->params.parserType == Lizard_parser_optimalPrice) { + Lizard_Insert(ctx, inr); + match_num = Lizard_GetAllMatches(ctx, inr, ip, matchlimit, best_mlen, matches); + LIZARD_LOG_PARSER("%d: Lizard_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); } else { - LIZ_BinTree_Insert(ctx, inr); - match_num = LIZ_BinTree_GetAllMatches(ctx, inr, matchlimit, best_mlen, matches); - LIZ_LOG_PARSER("%d: LIZ_BinTree_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); + Lizard_BinTree_Insert(ctx, inr); + match_num = Lizard_BinTree_GetAllMatches(ctx, inr, matchlimit, best_mlen, matches); + LIZARD_LOG_PARSER("%d: Lizard_BinTree_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); } @@ -584,8 +584,8 @@ FORCE_INLINE int LIZ_compress_optimalPrice( for (i = 0; i < match_num; i++) { mlen = (i>0) ? (size_t)matches[i-1].len+1 : best_mlen; cur2 = cur - matches[i].back; - best_mlen = (cur2 + matches[i].len < LIZ_OPT_NUM) ? (size_t)matches[i].len : LIZ_OPT_NUM - cur2; - LIZ_LOG_PARSER("%d: Found1 cur=%d cur2=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-source), cur, cur2, matches[i].len, matches[i].off, best_mlen, last_pos); + best_mlen = (cur2 + matches[i].len < LIZARD_OPT_NUM) ? (size_t)matches[i].len : LIZARD_OPT_NUM - cur2; + LIZARD_LOG_PARSER("%d: Found1 cur=%d cur2=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-source), cur, cur2, matches[i].len, matches[i].off, best_mlen, last_pos); if (mlen < (size_t)matches[i].back + 1) mlen = matches[i].back + 1; @@ -597,18 +597,18 @@ FORCE_INLINE int LIZ_compress_optimalPrice( litlen = opt[cur2].litlen; if (cur2 != litlen) - price = opt[cur2 - litlen].price + LIZ_get_price(ctx, rep, inr, opt[cur2].off24pos, litlen, matches[i].off, mlen); + price = opt[cur2 - litlen].price + Lizard_get_price(ctx, rep, inr, opt[cur2].off24pos, litlen, matches[i].off, mlen); else - price = LIZ_get_price(ctx, rep, inr, ctx->off24pos, llen + litlen, matches[i].off, mlen); + price = Lizard_get_price(ctx, rep, inr, ctx->off24pos, llen + litlen, matches[i].off, mlen); } else { litlen = 0; - price = opt[cur2].price + LIZ_get_price(ctx, rep, inr, opt[cur2].off24pos, litlen, matches[i].off, mlen); + price = opt[cur2].price + Lizard_get_price(ctx, rep, inr, opt[cur2].off24pos, litlen, matches[i].off, mlen); } - LIZ_LOG_PARSER("%d: Found2 pred=%d mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), matches[i].back, mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price); + LIZARD_LOG_PARSER("%d: Found2 pred=%d mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), matches[i].back, mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price); // if (cur2 + mlen > last_pos || ((matches[i].off != opt[cur2 + mlen].off) && (price < opt[cur2 + mlen].price))) - if ((mlen >= minMatchLongOff) || (matches[i].off < LIZ_MAX_16BIT_OFFSET)) + if ((mlen >= minMatchLongOff) || (matches[i].off < LIZARD_MAX_16BIT_OFFSET)) if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) { SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price); @@ -626,10 +626,10 @@ FORCE_INLINE int LIZ_compress_optimalPrice( encode: // cur, last_pos, best_mlen, best_off have to be set for (i = 1; i <= last_pos; i++) { - LIZ_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen, opt[i].rep); + LIZARD_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen, opt[i].rep); } - LIZ_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep=%d\n", (int)(ip-source+cur), cur, last_pos, best_mlen, best_off, opt[cur].rep); + LIZARD_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep=%d\n", (int)(ip-source+cur), cur, last_pos, best_mlen, best_off, opt[cur].rep); opt[0].mlen = 1; @@ -645,31 +645,31 @@ FORCE_INLINE int LIZ_compress_optimalPrice( } for (i = 0; i <= last_pos;) { - LIZ_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen, opt[i].rep); + LIZARD_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen, opt[i].rep); i += opt[i].mlen; } cur = 0; while (cur < last_pos) { - LIZ_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d\n", (int)(ip-source+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep); + LIZARD_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d\n", (int)(ip-source+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep); mlen = opt[cur].mlen; // if (mlen == 1) { ip++; cur++; continue; } if (opt[cur].off == -1) { ip++; cur++; continue; } offset = opt[cur].off; cur += mlen; - LIZ_LOG_ENCODE("%d: ENCODE literals=%d off=%d mlen=%d ", (int)(ip-source), (int)(ip-anchor), (int)(offset), mlen); - res = LIZ_encodeSequence(ctx, &ip, &anchor, mlen, ip - offset); + LIZARD_LOG_ENCODE("%d: ENCODE literals=%d off=%d mlen=%d ", (int)(ip-source), (int)(ip-anchor), (int)(offset), mlen); + res = Lizard_encodeSequence(ctx, &ip, &anchor, mlen, ip - offset); if (res) return 0; - LIZ_LOG_PARSER("%d: offset=%d rep=%d\n", (int)(ip-source), offset, ctx->last_off); + LIZARD_LOG_PARSER("%d: offset=%d rep=%d\n", (int)(ip-source), offset, ctx->last_off); } } /* Encode Last Literals */ ip = iend; - if (LIZ_encodeLastLiterals(ctx, &ip, &anchor)) goto _output_error; + if (Lizard_encodeLastLiterals(ctx, &ip, &anchor)) goto _output_error; /* End */ return 1; diff --git a/C/lizard/liz_parser_pricefast.h b/C/lizard/lizard_parser_pricefast.h similarity index 69% rename from C/lizard/liz_parser_pricefast.h rename to C/lizard/lizard_parser_pricefast.h index 1a914d12..9818e2c4 100644 --- a/C/lizard/liz_parser_pricefast.h +++ b/C/lizard/lizard_parser_pricefast.h @@ -1,6 +1,6 @@ -#define LIZ_PRICEFAST_MIN_OFFSET 8 +#define LIZARD_PRICEFAST_MIN_OFFSET 8 -FORCE_INLINE int LIZ_FindMatchFast(LIZ_stream_t* ctx, intptr_t matchIndex, intptr_t matchIndex3, /* Index table will be updated */ +FORCE_INLINE int Lizard_FindMatchFast(Lizard_stream_t* ctx, intptr_t matchIndex, intptr_t matchIndex3, /* Index table will be updated */ const BYTE* ip, const BYTE* const iLimit, const BYTE** matchpos) { @@ -16,14 +16,14 @@ FORCE_INLINE int LIZ_FindMatchFast(LIZ_stream_t* ctx, intptr_t matchIndex, intpt const BYTE* match, *matchDict; size_t ml=0, mlt; - if (ctx->last_off >= LIZ_PRICEFAST_MIN_OFFSET) { + if (ctx->last_off >= LIZARD_PRICEFAST_MIN_OFFSET) { intptr_t matchIndexLO = (ip - ctx->last_off) - base; if (matchIndexLO >= lowLimit) { if (matchIndexLO >= dictLimit) { match = base + matchIndexLO; if (MEM_readMINMATCH(match) == MEM_readMINMATCH(ip)) { - mlt = LIZ_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; - // if ((mlt >= minMatchLongOff) || (ctx->last_off < LIZ_MAX_16BIT_OFFSET)) + mlt = Lizard_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; + // if ((mlt >= minMatchLongOff) || (ctx->last_off < LIZARD_MAX_16BIT_OFFSET)) { *matchpos = match; return (int)mlt; @@ -33,8 +33,8 @@ FORCE_INLINE int LIZ_FindMatchFast(LIZ_stream_t* ctx, intptr_t matchIndex, intpt match = dictBase + matchIndexLO; if ((U32)((dictLimit-1) - matchIndexLO) >= 3) /* intentional overflow */ if (MEM_readMINMATCH(match) == MEM_readMINMATCH(ip)) { - mlt = LIZ_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH; - // if ((mlt >= minMatchLongOff) || (ctx->last_off < LIZ_MAX_16BIT_OFFSET)) + mlt = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH; + // if ((mlt >= minMatchLongOff) || (ctx->last_off < LIZARD_MAX_16BIT_OFFSET)) { *matchpos = base + matchIndexLO; /* virtual matchpos */ return (int)mlt; @@ -48,10 +48,10 @@ FORCE_INLINE int LIZ_FindMatchFast(LIZ_stream_t* ctx, intptr_t matchIndex, intpt #if MINMATCH == 3 if (matchIndex3 < current && matchIndex3 >= lowLimit) { intptr_t offset = current - matchIndex3; - if (offset < LIZ_MAX_8BIT_OFFSET) { + if (offset < LIZARD_MAX_8BIT_OFFSET) { match = ip - offset; if (match > base && MEM_readMINMATCH(ip) == MEM_readMINMATCH(match)) { - ml = 3;//LIZ_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; + ml = 3;//Lizard_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; *matchpos = match; } } @@ -62,22 +62,21 @@ FORCE_INLINE int LIZ_FindMatchFast(LIZ_stream_t* ctx, intptr_t matchIndex, intpt if ((matchIndex < current) && (matchIndex >= lowLimit)) { match = base + matchIndex; - if ((U32)(ip - match) >= LIZ_PRICEFAST_MIN_OFFSET) { + if ((U32)(ip - match) >= LIZARD_PRICEFAST_MIN_OFFSET) { if (matchIndex >= dictLimit) { if (*(match+ml) == *(ip+ml) && (MEM_read32(match) == MEM_read32(ip))) { - mlt = LIZ_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; - if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZ_MAX_16BIT_OFFSET)) - if (!ml || (mlt > ml)) // && LIZ_better_price((ip - *matchpos), ml, (ip - match), mlt, ctx->last_off))) + mlt = Lizard_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; + if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) + if (!ml || (mlt > ml)) // && Lizard_better_price((ip - *matchpos), ml, (ip - match), mlt, ctx->last_off))) { ml = mlt; *matchpos = match; } } } else { matchDict = dictBase + matchIndex; - // fprintf(stderr, "dictBase[%p]+matchIndex[%d]=match[%p] dictLimit=%d base=%p ip=%p iLimit=%p off=%d\n", dictBase, matchIndex, match, dictLimit, base, ip, iLimit, (U32)(ip-match)); if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ if (MEM_read32(matchDict) == MEM_read32(ip)) { - mlt = LIZ_count_2segments(ip+MINMATCH, matchDict+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH; - if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZ_MAX_16BIT_OFFSET)) - if (!ml || (mlt > ml)) // && LIZ_better_price((ip - *matchpos), ml, (U32)(ip - match), mlt, ctx->last_off))) + mlt = Lizard_count_2segments(ip+MINMATCH, matchDict+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH; + if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) + if (!ml || (mlt > ml)) // && Lizard_better_price((ip - *matchpos), ml, (U32)(ip - match), mlt, ctx->last_off))) { ml = mlt; *matchpos = match; } /* virtual matchpos */ } } @@ -88,7 +87,7 @@ FORCE_INLINE int LIZ_FindMatchFast(LIZ_stream_t* ctx, intptr_t matchIndex, intpt } -FORCE_INLINE int LIZ_FindMatchFaster (LIZ_stream_t* ctx, U32 matchIndex, /* Index table will be updated */ +FORCE_INLINE int Lizard_FindMatchFaster (Lizard_stream_t* ctx, U32 matchIndex, /* Index table will be updated */ const BYTE* ip, const BYTE* const iLimit, const BYTE** matchpos) { @@ -106,19 +105,19 @@ FORCE_INLINE int LIZ_FindMatchFaster (LIZ_stream_t* ctx, U32 matchIndex, /* Ind if (matchIndex < current && matchIndex >= lowLimit) { match = base + matchIndex; - if ((U32)(ip - match) >= LIZ_PRICEFAST_MIN_OFFSET) { + if ((U32)(ip - match) >= LIZARD_PRICEFAST_MIN_OFFSET) { if (matchIndex >= dictLimit) { if (MEM_read32(match) == MEM_read32(ip)) { - mlt = LIZ_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; - if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZ_MAX_16BIT_OFFSET)) + mlt = Lizard_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; + if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) { ml = mlt; *matchpos = match; } } } else { matchDict = dictBase + matchIndex; if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ if (MEM_read32(matchDict) == MEM_read32(ip)) { - mlt = LIZ_count_2segments(ip+MINMATCH, matchDict+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH; - if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZ_MAX_16BIT_OFFSET)) + mlt = Lizard_count_2segments(ip+MINMATCH, matchDict+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH; + if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) { ml = mlt; *matchpos = match; } /* virtual matchpos */ } } @@ -130,8 +129,8 @@ FORCE_INLINE int LIZ_FindMatchFaster (LIZ_stream_t* ctx, U32 matchIndex, /* Ind -FORCE_INLINE int LIZ_compress_priceFast( - LIZ_stream_t* const ctx, +FORCE_INLINE int Lizard_compress_priceFast( + Lizard_stream_t* const ctx, const BYTE* ip, const BYTE* const iend) { @@ -158,17 +157,17 @@ FORCE_INLINE int LIZ_compress_priceFast( /* Main Loop */ while (ip < mflimit) { - HashPos = &HashTable[LIZ_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)]; + HashPos = &HashTable[Lizard_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)]; #if MINMATCH == 3 { - U32* HashPos3 = &HashTable3[LIZ_hash3Ptr(ip, ctx->params.hashLog3)]; - ml = LIZ_FindMatchFast (ctx, *HashPos, *HashPos3, ip, matchlimit, (&ref)); + U32* HashPos3 = &HashTable3[Lizard_hash3Ptr(ip, ctx->params.hashLog3)]; + ml = Lizard_FindMatchFast (ctx, *HashPos, *HashPos3, ip, matchlimit, (&ref)); *HashPos3 = (U32)(ip - base); } #else - ml = LIZ_FindMatchFast (ctx, *HashPos, 0, ip, matchlimit, (&ref)); + ml = Lizard_FindMatchFast (ctx, *HashPos, 0, ip, matchlimit, (&ref)); #endif - if ((U32)(ip - base) >= *HashPos + LIZ_PRICEFAST_MIN_OFFSET) + if ((*HashPos >= (U32)(ip - base)) || ((U32)(ip - base) >= *HashPos + LIZARD_PRICEFAST_MIN_OFFSET)) *HashPos = (U32)(ip - base); if (!ml) { ip++; continue; } @@ -186,9 +185,9 @@ _Search: if (ip+ml >= mflimit) goto _Encode; start2 = ip + ml - 2; - HashPos = &HashTable[LIZ_hashPtr(start2, ctx->params.hashLog, ctx->params.searchLength)]; - ml2 = LIZ_FindMatchFaster(ctx, *HashPos, start2, matchlimit, (&ref2)); - if ((U32)(start2 - base) >= *HashPos + LIZ_PRICEFAST_MIN_OFFSET) + HashPos = &HashTable[Lizard_hashPtr(start2, ctx->params.hashLog, ctx->params.searchLength)]; + ml2 = Lizard_FindMatchFaster(ctx, *HashPos, start2, matchlimit, (&ref2)); + if ((*HashPos >= (U32)(start2 - base)) || ((U32)(start2 - base) >= *HashPos + LIZARD_PRICEFAST_MIN_OFFSET)) *HashPos = (U32)(start2 - base); if (!ml2) goto _Encode; @@ -201,8 +200,6 @@ _Search: ref2 += back; } - // LIZ_DEBUG("%u: TRY last_off=%d literals=%u off=%u mlen=%u literals2=%u off2=%u mlen2=%u best=%d\n", (U32)(ip - ctx->inputBuffer), ctx->last_off, (U32)(ip - anchor), off0, (U32)ml, (U32)(start2 - anchor), off1, ml2, (U32)(best_pos - ip)); - if (ml2 <= ml) { ml2 = 0; goto _Encode; } if (start2 <= ip) @@ -227,11 +224,11 @@ _Search: ref2 += correction; ml2 -= correction; if (ml2 < 3) { ml2 = 0; } - if ((ml2 < minMatchLongOff) && ((U32)(start2 - ref2) >= LIZ_MAX_16BIT_OFFSET)) { ml2 = 0; } + if ((ml2 < minMatchLongOff) && ((U32)(start2 - ref2) >= LIZARD_MAX_16BIT_OFFSET)) { ml2 = 0; } } _Encode: - if (LIZ_encodeSequence_LZ5v2(ctx, &ip, &anchor, ml, ref)) goto _output_error; + if (Lizard_encodeSequence_LIZv1(ctx, &ip, &anchor, ml, ref)) goto _output_error; if (ml2) { @@ -243,7 +240,7 @@ _Encode: /* Encode Last Literals */ ip = iend; - if (LIZ_encodeLastLiterals_LZ5v2(ctx, &ip, &anchor)) goto _output_error; + if (Lizard_encodeLastLiterals_LIZv1(ctx, &ip, &anchor)) goto _output_error; /* End */ return 1; diff --git a/C/lizard/mem.h b/C/lizard/mem.h deleted file mode 100644 index 1b0af312..00000000 --- a/C/lizard/mem.h +++ /dev/null @@ -1,394 +0,0 @@ -/* ****************************************************************** - mem.h - low-level memory access routines - Copyright (C) 2013-2015, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy -****************************************************************** */ -#ifndef MEM_H_MODULE -#define MEM_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - -/*-**************************************** -* Dependencies -******************************************/ -#include /* size_t, ptrdiff_t */ -#include /* memcpy */ - - -/*-**************************************** -* Compiler specifics -******************************************/ -#if defined(_MSC_VER) /* Visual Studio */ -# include /* _byteswap_ulong */ -# include /* _byteswap_* */ -#endif -#if defined(__GNUC__) -# define MEM_STATIC static __inline __attribute__((unused)) -#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define MEM_STATIC static inline -#elif defined(_MSC_VER) -# define MEM_STATIC static __inline -#else -# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ -#endif - -/* code only tested on 32 and 64 bits systems */ -#define MEM_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } -MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } - - -/*-************************************************************** -* Basic Types -*****************************************************************/ -#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef int16_t S16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; - typedef int64_t S64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef signed short S16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; - typedef signed long long S64; -#endif - - -/*-************************************************************** -* Memory I/O -*****************************************************************/ -/* MEM_FORCE_MEMORY_ACCESS : - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. - * Method 2 : direct access. This method is portable but violate C standard. - * It can generate buggy code on targets depending on alignment. - * In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) - * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. - * Prefer these methods in priority order (0 > 1 > 2) - */ -#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) -# define MEM_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) /*|| defined(_MSC_VER)*/ || \ - (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) -# define MEM_FORCE_MEMORY_ACCESS 1 -# endif -#endif - -MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; } -MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } - -MEM_STATIC unsigned MEM_isLittleEndian(void) -{ - const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ - return one.c[0]; -} - -#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) - -/* violates C standard, by lying on structure alignment. -Only use if no other choice to achieve best performance on target platform */ -MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } -MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } -MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } -MEM_STATIC U64 MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; } - -MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } -MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } -MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } - -#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) - -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32)) - __pragma( pack(push, 1) ) - typedef union { U16 u16; U32 u32; U64 u64; size_t st; } unalign; - __pragma( pack(pop) ) -#else - typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign; -#endif - -MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } -MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } -MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } -MEM_STATIC U64 MEM_readST(const void* ptr) { return ((const unalign*)ptr)->st; } - -MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } -MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } -MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; } - -#else - -/* default method, safe and standard. - can sometimes prove slower */ - -MEM_STATIC U16 MEM_read16(const void* memPtr) -{ - U16 val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -MEM_STATIC U32 MEM_read32(const void* memPtr) -{ - U32 val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -MEM_STATIC U64 MEM_read64(const void* memPtr) -{ - U64 val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -MEM_STATIC size_t MEM_readST(const void* memPtr) -{ - size_t val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -MEM_STATIC void MEM_write16(void* memPtr, U16 value) -{ - memcpy(memPtr, &value, sizeof(value)); -} - -MEM_STATIC void MEM_write32(void* memPtr, U32 value) -{ - memcpy(memPtr, &value, sizeof(value)); -} - -MEM_STATIC void MEM_write64(void* memPtr, U64 value) -{ - memcpy(memPtr, &value, sizeof(value)); -} - -#endif /* MEM_FORCE_MEMORY_ACCESS */ - -MEM_STATIC U32 MEM_swap32(U32 in) -{ -#if defined(_MSC_VER) /* Visual Studio */ - return _byteswap_ulong(in); -#elif defined (__GNUC__) - return __builtin_bswap32(in); -#else - return ((in << 24) & 0xff000000 ) | - ((in << 8) & 0x00ff0000 ) | - ((in >> 8) & 0x0000ff00 ) | - ((in >> 24) & 0x000000ff ); -#endif -} - -MEM_STATIC U64 MEM_swap64(U64 in) -{ -#if defined(_MSC_VER) /* Visual Studio */ - return _byteswap_uint64(in); -#elif defined (__GNUC__) - return __builtin_bswap64(in); -#else - return ((in << 56) & 0xff00000000000000ULL) | - ((in << 40) & 0x00ff000000000000ULL) | - ((in << 24) & 0x0000ff0000000000ULL) | - ((in << 8) & 0x000000ff00000000ULL) | - ((in >> 8) & 0x00000000ff000000ULL) | - ((in >> 24) & 0x0000000000ff0000ULL) | - ((in >> 40) & 0x000000000000ff00ULL) | - ((in >> 56) & 0x00000000000000ffULL); -#endif -} - -MEM_STATIC size_t MEM_swapST(size_t in) -{ - if (MEM_32bits()) - return (size_t)MEM_swap32((U32)in); - else - return (size_t)MEM_swap64((U64)in); -} - -/*=== Little endian r/w ===*/ - -MEM_STATIC U16 MEM_readLE16(const void* memPtr) -{ - if (MEM_isLittleEndian()) - return MEM_read16(memPtr); - else { - const BYTE* p = (const BYTE*)memPtr; - return (U16)(p[0] + (p[1]<<8)); - } -} - -MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) -{ - if (MEM_isLittleEndian()) { - MEM_write16(memPtr, val); - } else { - BYTE* p = (BYTE*)memPtr; - p[0] = (BYTE)val; - p[1] = (BYTE)(val>>8); - } -} - -MEM_STATIC U32 MEM_readLE24(const void* memPtr) -{ - return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16); -} - -MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) -{ - MEM_writeLE16(memPtr, (U16)val); - ((BYTE*)memPtr)[2] = (BYTE)(val>>16); -} - -MEM_STATIC U32 MEM_readLE32(const void* memPtr) -{ - if (MEM_isLittleEndian()) - return MEM_read32(memPtr); - else - return MEM_swap32(MEM_read32(memPtr)); -} - -MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32) -{ - if (MEM_isLittleEndian()) - MEM_write32(memPtr, val32); - else - MEM_write32(memPtr, MEM_swap32(val32)); -} - -MEM_STATIC U64 MEM_readLE64(const void* memPtr) -{ - if (MEM_isLittleEndian()) - return MEM_read64(memPtr); - else - return MEM_swap64(MEM_read64(memPtr)); -} - -MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64) -{ - if (MEM_isLittleEndian()) - MEM_write64(memPtr, val64); - else - MEM_write64(memPtr, MEM_swap64(val64)); -} - -MEM_STATIC size_t MEM_readLEST(const void* memPtr) -{ - if (MEM_32bits()) - return (size_t)MEM_readLE32(memPtr); - else - return (size_t)MEM_readLE64(memPtr); -} - -MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val) -{ - if (MEM_32bits()) - MEM_writeLE32(memPtr, (U32)val); - else - MEM_writeLE64(memPtr, (U64)val); -} - -/*=== Big endian r/w ===*/ - -MEM_STATIC U32 MEM_readBE32(const void* memPtr) -{ - if (MEM_isLittleEndian()) - return MEM_swap32(MEM_read32(memPtr)); - else - return MEM_read32(memPtr); -} - -MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32) -{ - if (MEM_isLittleEndian()) - MEM_write32(memPtr, MEM_swap32(val32)); - else - MEM_write32(memPtr, val32); -} - -MEM_STATIC U64 MEM_readBE64(const void* memPtr) -{ - if (MEM_isLittleEndian()) - return MEM_swap64(MEM_read64(memPtr)); - else - return MEM_read64(memPtr); -} - -MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64) -{ - if (MEM_isLittleEndian()) - MEM_write64(memPtr, MEM_swap64(val64)); - else - MEM_write64(memPtr, val64); -} - -MEM_STATIC size_t MEM_readBEST(const void* memPtr) -{ - if (MEM_32bits()) - return (size_t)MEM_readBE32(memPtr); - else - return (size_t)MEM_readBE64(memPtr); -} - -MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) -{ - if (MEM_32bits()) - MEM_writeBE32(memPtr, (U32)val); - else - MEM_writeBE64(memPtr, (U64)val); -} - - -/* function safe only for comparisons */ -MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length) -{ - switch (length) - { - default : - case 4 : return MEM_read32(memPtr); - case 3 : if (MEM_isLittleEndian()) - return MEM_read32(memPtr)<<8; - else - return MEM_read32(memPtr)>>8; - } -} - -#if defined (__cplusplus) -} -#endif - -#endif /* MEM_H_MODULE */ diff --git a/C/zstdmt/lizard-mt_common.c b/C/zstdmt/lizard-mt_common.c index 9f42ca8b..598a0d36 100644 --- a/C/zstdmt/lizard-mt_common.c +++ b/C/zstdmt/lizard-mt_common.c @@ -9,7 +9,7 @@ * of patent rights can be found in the PATENTS file in the same directory. */ -#include "lizframe.h" +#include "lizard_frame.h" #include "lizard-mt.h" /* will be used for lib errors */ @@ -32,8 +32,8 @@ unsigned LIZARDMT_isError(size_t code) */ const char *LIZARDMT_getErrorString(size_t code) { - if (LIZF_isError(lizardmt_errcode)) - return LIZF_getErrorName(lizardmt_errcode); + if (LizardF_isError(lizardmt_errcode)) + return LizardF_getErrorName(lizardmt_errcode); static const char *noErrorCode = "Unspecified lizardmt error code"; switch ((LIZARDMT_ErrorCode) (0 - code)) { diff --git a/C/zstdmt/lizard-mt_compress.c b/C/zstdmt/lizard-mt_compress.c index 9dfdd038..696d8f55 100644 --- a/C/zstdmt/lizard-mt_compress.c +++ b/C/zstdmt/lizard-mt_compress.c @@ -14,8 +14,8 @@ #include #include -#define LIZF_DISABLE_OBSOLETE_ENUMS -#include "lizframe.h" +#define LizardF_DISABLE_OBSOLETE_ENUMS +#include "lizard_frame.h" #include "memmt.h" #include "threading.h" @@ -38,7 +38,7 @@ /* worker for compression */ typedef struct { LIZARDMT_CCtx *ctx; - LIZF_preferences_t zpref; + LizardF_preferences_t zpref; pthread_t pthread; } cwork_t; @@ -138,12 +138,12 @@ LIZARDMT_CCtx *LIZARDMT_createCCtx(int threads, int level, int inputsize) w->ctx = ctx; /* setup preferences for that thread */ - memset(&w->zpref, 0, sizeof(LIZF_preferences_t)); + memset(&w->zpref, 0, sizeof(LizardF_preferences_t)); w->zpref.compressionLevel = level; - w->zpref.frameInfo.blockMode = LIZF_blockLinked; + w->zpref.frameInfo.blockMode = LizardF_blockLinked; w->zpref.frameInfo.contentSize = 1; w->zpref.frameInfo.contentChecksumFlag = - LIZF_contentChecksumEnabled; + LizardF_contentChecksumEnabled; } @@ -229,7 +229,7 @@ static void *pt_compress(void *arg) entry = list_first(&ctx->writelist_free); wl = list_entry(entry, struct writelist, node); wl->out.size = - LIZF_compressFrameBound(ctx->inputsize, + LizardF_compressFrameBound(ctx->inputsize, &w->zpref) + 12; list_move(entry, &ctx->writelist_busy); } else { @@ -241,7 +241,7 @@ static void *pt_compress(void *arg) return (void *)ERROR(memory_allocation); } wl->out.size = - LIZF_compressFrameBound(ctx->inputsize, + LizardF_compressFrameBound(ctx->inputsize, &w->zpref) + 12;; wl->out.buf = malloc(wl->out.size); if (!wl->out.buf) { @@ -278,10 +278,10 @@ static void *pt_compress(void *arg) /* compress whole frame */ result = - LIZF_compressFrame((unsigned char *)wl->out.buf + 12, + LizardF_compressFrame((unsigned char *)wl->out.buf + 12, wl->out.size - 12, in.buf, in.size, &w->zpref); - if (LIZF_isError(result)) { + if (LizardF_isError(result)) { pthread_mutex_lock(&ctx->write_mutex); list_move(&wl->node, &ctx->writelist_free); pthread_mutex_unlock(&ctx->write_mutex); diff --git a/C/zstdmt/lizard-mt_decompress.c b/C/zstdmt/lizard-mt_decompress.c index 6f4e91cc..339aa568 100644 --- a/C/zstdmt/lizard-mt_decompress.c +++ b/C/zstdmt/lizard-mt_decompress.c @@ -14,8 +14,8 @@ #include #include -#define LIZF_DISABLE_OBSOLETE_ENUMS -#include "lizframe.h" +#define LizardF_DISABLE_OBSOLETE_ENUMS +#include "lizard_frame.h" #include "memmt.h" #include "threading.h" @@ -40,7 +40,7 @@ typedef struct { LIZARDMT_DCtx *ctx; pthread_t pthread; LIZARDMT_Buffer in; - LIZF_decompressionContext_t dctx; + LizardF_decompressionContext_t dctx; } cwork_t; struct writelist; @@ -130,7 +130,7 @@ LIZARDMT_DCtx *LIZARDMT_createDCtx(int threads, int inputsize) w->ctx = ctx; /* setup thread work */ - LIZF_createDecompressionContext(&w->dctx, LIZF_VERSION); + LizardF_createDecompressionContext(&w->dctx, LIZARDF_VERSION); } return ctx; @@ -348,10 +348,10 @@ static void *pt_decompress(void *arg) } result = - LIZF_decompress(w->dctx, out->buf, &out->size, + LizardF_decompress(w->dctx, out->buf, &out->size, in->buf, &in->size, 0); - if (LIZF_isError(result)) { + if (LizardF_isError(result)) { lizardmt_errcode = result; result = ERROR(compression_library); goto error_lock; @@ -392,7 +392,7 @@ static void *pt_decompress(void *arg) static size_t st_decompress(void *arg) { LIZARDMT_DCtx *ctx = (LIZARDMT_DCtx *) arg; - LIZF_errorCode_t nextToLoad = 0; + LizardF_errorCode_t nextToLoad = 0; cwork_t *w = &ctx->cwork[0]; LIZARDMT_Buffer Out; LIZARDMT_Buffer *out = &Out; @@ -420,8 +420,8 @@ static size_t st_decompress(void *arg) memcpy(in->buf, magic, in->size); nextToLoad = - LIZF_decompress(w->dctx, out->buf, &pos, in->buf, &in->size, 0); - if (LIZF_isError(nextToLoad)) { + LizardF_decompress(w->dctx, out->buf, &pos, in->buf, &in->size, 0); + if (LizardF_isError(nextToLoad)) { free(in->buf); free(out->buf); return ERROR(compression_library); @@ -451,10 +451,10 @@ static size_t st_decompress(void *arg) /* decompress */ nextToLoad = - LIZF_decompress(w->dctx, out->buf, &out->size, + LizardF_decompress(w->dctx, out->buf, &out->size, (unsigned char *)in->buf + pos, &remaining, NULL); - if (LIZF_isError(nextToLoad)) { + if (LizardF_isError(nextToLoad)) { free(in->buf); free(out->buf); return ERROR(compression_library); @@ -603,7 +603,7 @@ void LIZARDMT_freeDCtx(LIZARDMT_DCtx * ctx) for (t = 0; t < ctx->threads; t++) { cwork_t *w = &ctx->cwork[t]; - LIZF_freeDecompressionContext(w->dctx); + LizardF_freeDecompressionContext(w->dctx); } pthread_mutex_destroy(&ctx->read_mutex); diff --git a/CPP/7zip/Archive/Icons/liz.ico b/CPP/7zip/Archive/Icons/liz.ico new file mode 100644 index 00000000..de41596c Binary files /dev/null and b/CPP/7zip/Archive/Icons/liz.ico differ diff --git a/CPP/7zip/Archive/LizardHandler.cpp b/CPP/7zip/Archive/LizardHandler.cpp new file mode 100644 index 00000000..682925e1 --- /dev/null +++ b/CPP/7zip/Archive/LizardHandler.cpp @@ -0,0 +1,366 @@ +// LizardHandler.cpp + +#include "StdAfx.h" + +#include "../../../C/CpuArch.h" +#include "../../Common/ComTry.h" +#include "../../Common/Defs.h" + +#include "../Common/ProgressUtils.h" +#include "../Common/RegisterArc.h" +#include "../Common/StreamUtils.h" + +#include "../Compress/LizardDecoder.h" +#include "../Compress/LizardEncoder.h" +#include "../Compress/CopyCoder.h" + +#include "Common/DummyOutStream.h" +#include "Common/HandlerOut.h" + +using namespace NWindows; + +namespace NArchive { +namespace NLIZARD { + +class CHandler: + public IInArchive, + public IArchiveOpenSeq, + public IOutArchive, + public ISetProperties, + public CMyUnknownImp +{ + CMyComPtr _stream; + CMyComPtr _seqStream; + + bool _isArc; + bool _dataAfterEnd; + bool _needMoreInput; + + bool _packSize_Defined; + bool _unpackSize_Defined; + + UInt64 _packSize; + UInt64 _unpackSize; + UInt64 _numStreams; + UInt64 _numBlocks; + + CSingleMethodProps _props; + +public: + MY_UNKNOWN_IMP4( + IInArchive, + IArchiveOpenSeq, + IOutArchive, + ISetProperties) + INTERFACE_IInArchive(;) + INTERFACE_IOutArchive(;) + STDMETHOD(OpenSeq)(ISequentialInStream *stream); + STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps); + + CHandler() { } +}; + +static const Byte kProps[] = +{ + kpidSize, + kpidPackSize +}; + +static const Byte kArcProps[] = +{ + kpidNumStreams, + kpidNumBlocks +}; + +IMP_IInArchive_Props +IMP_IInArchive_ArcProps + +STDMETHODIMP CHandler::GetArchiveProperty(PROPID /*propID*/, PROPVARIANT * /*value*/) +{ + return S_OK; +} + +STDMETHODIMP CHandler::GetNumberOfItems(UInt32 *numItems) +{ + *numItems = 1; + return S_OK; +} + +STDMETHODIMP CHandler::GetProperty(UInt32 /* index */, PROPID propID, PROPVARIANT *value) +{ + NCOM::CPropVariant prop; + switch (propID) + { + case kpidPackSize: if (_packSize_Defined) prop = _packSize; break; + case kpidSize: if (_unpackSize_Defined) prop = _unpackSize; break; + } + prop.Detach(value); + return S_OK; +} + +static const unsigned kSignatureCheckSize = 4; + +API_FUNC_static_IsArc IsArc_lizard(const Byte *p, size_t size) +{ + if (size < 4) + return k_IsArc_Res_NEED_MORE; + + UInt32 magic = GetUi32(p); + + // skippable frames + if (magic >= 0x184D2A50 && magic <= 0x184D2A5F) { + if (size < 16) + return k_IsArc_Res_NEED_MORE; + magic = GetUi32(p+12); + } + + // Lizard Magic + if (magic == 0x184D2206) + return k_IsArc_Res_YES; + + return k_IsArc_Res_NO; +} +} + +STDMETHODIMP CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallback *) +{ + COM_TRY_BEGIN + Close(); + { + Byte buf[kSignatureCheckSize]; + RINOK(ReadStream_FALSE(stream, buf, kSignatureCheckSize)); + if (IsArc_lizard(buf, kSignatureCheckSize) == k_IsArc_Res_NO) + return S_FALSE; + _isArc = true; + _stream = stream; + _seqStream = stream; + RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL)); + } + return S_OK; + COM_TRY_END +} + +STDMETHODIMP CHandler::OpenSeq(ISequentialInStream *stream) +{ + Close(); + _isArc = true; + _seqStream = stream; + return S_OK; +} + +STDMETHODIMP CHandler::Close() +{ + _isArc = false; + _dataAfterEnd = false; + _needMoreInput = false; + + _packSize_Defined = false; + _unpackSize_Defined = false; + + _packSize = 0; + + _seqStream.Release(); + _stream.Release(); + return S_OK; +} + +STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, + Int32 testMode, IArchiveExtractCallback *extractCallback) +{ + COM_TRY_BEGIN + if (numItems == 0) + return S_OK; + if (numItems != (UInt32)(Int32)-1 && (numItems != 1 || indices[0] != 0)) + return E_INVALIDARG; + + if (_packSize_Defined) + extractCallback->SetTotal(_packSize); + + CMyComPtr realOutStream; + Int32 askMode = testMode ? + NExtract::NAskMode::kTest : + NExtract::NAskMode::kExtract; + RINOK(extractCallback->GetStream(0, &realOutStream, askMode)); + if (!testMode && !realOutStream) + return S_OK; + + extractCallback->PrepareOperation(askMode); + + Int32 opRes; + + { + + NCompress::NLIZARD::CDecoder *decoderSpec = new NCompress::NLIZARD::CDecoder; + CMyComPtr decoder = decoderSpec; + decoderSpec->SetInStream(_seqStream); + + CDummyOutStream *outStreamSpec = new CDummyOutStream; + CMyComPtr outStream(outStreamSpec); + outStreamSpec->SetStream(realOutStream); + outStreamSpec->Init(); + + realOutStream.Release(); + + CLocalProgress *lps = new CLocalProgress; + CMyComPtr progress = lps; + lps->Init(extractCallback, true); + + UInt64 packSize = 0; + UInt64 unpackedSize = 0; + + HRESULT result = S_OK; + + for (;;) + { + lps->InSize = packSize; + lps->OutSize = unpackedSize; + + RINOK(lps->SetCur()); + result = decoderSpec->CodeResume(outStream, &unpackedSize, progress); + UInt64 streamSize = decoderSpec->GetInputProcessedSize(); + + if (result != S_FALSE && result != S_OK) + return result; + + if (unpackedSize == 0) + break; + + if (streamSize == packSize) + { + // no new bytes in input stream, So it's good end of archive. + result = S_OK; + break; + } + + if (packSize > streamSize) + return E_FAIL; + + if (result != S_OK) + break; + } + + decoderSpec->ReleaseInStream(); + outStream.Release(); + + if (!_isArc) + opRes = NExtract::NOperationResult::kIsNotArc; + else if (_needMoreInput) + opRes = NExtract::NOperationResult::kUnexpectedEnd; + else if (_dataAfterEnd) + opRes = NExtract::NOperationResult::kDataAfterEnd; + else if (result == S_FALSE) + opRes = NExtract::NOperationResult::kDataError; + else if (result == S_OK) + opRes = NExtract::NOperationResult::kOK; + else + return result; + + } + + return extractCallback->SetOperationResult(opRes); + + COM_TRY_END +} + +static HRESULT UpdateArchive( + UInt64 unpackSize, + ISequentialOutStream *outStream, + const CProps &props, + IArchiveUpdateCallback *updateCallback) +{ + RINOK(updateCallback->SetTotal(unpackSize)); + CMyComPtr fileInStream; + RINOK(updateCallback->GetStream(0, &fileInStream)); + CLocalProgress *localProgressSpec = new CLocalProgress; + CMyComPtr localProgress = localProgressSpec; + localProgressSpec->Init(updateCallback, true); + NCompress::NLIZARD::CEncoder *encoderSpec = new NCompress::NLIZARD::CEncoder; + CMyComPtr encoder = encoderSpec; + RINOK(props.SetCoderProps(encoderSpec, NULL)); + RINOK(encoder->Code(fileInStream, outStream, NULL, NULL, localProgress)); + return updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK); +} + +STDMETHODIMP CHandler::GetFileTimeType(UInt32 *type) +{ + *type = NFileTimeType::kUnix; + return S_OK; +} + +STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numItems, + IArchiveUpdateCallback *updateCallback) +{ + COM_TRY_BEGIN + + if (numItems != 1) + return E_INVALIDARG; + + Int32 newData, newProps; + UInt32 indexInArchive; + if (!updateCallback) + return E_FAIL; + RINOK(updateCallback->GetUpdateItemInfo(0, &newData, &newProps, &indexInArchive)); + + if ((newProps)) + { + { + NCOM::CPropVariant prop; + RINOK(updateCallback->GetProperty(0, kpidIsDir, &prop)); + if (prop.vt != VT_EMPTY) + if (prop.vt != VT_BOOL || prop.boolVal != VARIANT_FALSE) + return E_INVALIDARG; + } + } + + if ((newData)) + { + UInt64 size; + { + NCOM::CPropVariant prop; + RINOK(updateCallback->GetProperty(0, kpidSize, &prop)); + if (prop.vt != VT_UI8) + return E_INVALIDARG; + size = prop.uhVal.QuadPart; + } + return UpdateArchive(size, outStream, _props, updateCallback); + } + + if (indexInArchive != 0) + return E_INVALIDARG; + + CLocalProgress *lps = new CLocalProgress; + CMyComPtr progress = lps; + lps->Init(updateCallback, true); + + CMyComPtr opCallback; + updateCallback->QueryInterface(IID_IArchiveUpdateCallbackFile, (void **)&opCallback); + if (opCallback) + { + RINOK(opCallback->ReportOperation( + NEventIndexType::kInArcIndex, 0, + NUpdateNotifyOp::kReplicate)) + } + + if (_stream) + RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL)); + + return NCompress::CopyStream(_stream, outStream, progress); + + COM_TRY_END +} + +STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps) +{ + return _props.SetProperties(names, values, numProps); +} + +static const Byte k_Signature[] = "0x184D2206"; + +REGISTER_ARC_IO( + "liz", "liz tliz", "* .tar", 0x11, + k_Signature, + 0, + NArcInfoFlags::kKeepName, + IsArc_lizard) + +}} diff --git a/CPP/7zip/Archive/Lz4Handler.cpp b/CPP/7zip/Archive/Lz4Handler.cpp index f709cae2..1c71867a 100644 --- a/CPP/7zip/Archive/Lz4Handler.cpp +++ b/CPP/7zip/Archive/Lz4Handler.cpp @@ -1,4 +1,4 @@ -// ZstdHandler.cpp +// Lz4Handler.cpp #include "StdAfx.h" diff --git a/CPP/7zip/Archive/Lz5Handler.cpp b/CPP/7zip/Archive/Lz5Handler.cpp index 49485ee1..cf55239d 100644 --- a/CPP/7zip/Archive/Lz5Handler.cpp +++ b/CPP/7zip/Archive/Lz5Handler.cpp @@ -1,4 +1,4 @@ -// ZstdHandler.cpp +// Lz5Handler.cpp #include "StdAfx.h" diff --git a/CPP/7zip/Bundles/Alone/makefile b/CPP/7zip/Bundles/Alone/makefile index 9aa23f0a..52e03006 100644 --- a/CPP/7zip/Bundles/Alone/makefile +++ b/CPP/7zip/Bundles/Alone/makefile @@ -260,14 +260,9 @@ BROTLI_OBJS = \ $O/br_utf8_util.obj \ LIZARD_OBJS = \ - $O/liz_entropy_common.obj \ - $O/liz_fse_compress.obj \ - $O/liz_fse_decompress.obj \ - $O/liz_huf_compress.obj \ - $O/liz_huf_decompress.obj \ - $O/liz_lz5_compress.obj \ - $O/liz_lz5_decompress.obj \ - $O/liz_lz5frame.obj \ + $O/lizard_compress.obj \ + $O/lizard_decompress.obj \ + $O/lizard_frame.obj \ LZ4_OBJS = \ $O\lz4.obj \ diff --git a/CPP/7zip/Bundles/Codec_lizard/makefile b/CPP/7zip/Bundles/Codec_lizard/makefile index fb534fea..63dd6036 100644 --- a/CPP/7zip/Bundles/Codec_lizard/makefile +++ b/CPP/7zip/Bundles/Codec_lizard/makefile @@ -22,17 +22,18 @@ COMPRESS_OBJS = $(COMPRESS_OBJS) \ $O\LizardRegister.obj \ LIZARD_OBJS = \ - $O/liz_entropy_common.obj \ - $O/liz_fse_compress.obj \ - $O/liz_fse_decompress.obj \ - $O/liz_huf_compress.obj \ - $O/liz_huf_decompress.obj \ - $O/liz_lz5_compress.obj \ - $O/liz_lz5_decompress.obj \ - $O/liz_lz5frame.obj \ + $O/lizard_compress.obj \ + $O/lizard_decompress.obj \ + $O/lizard_frame.obj \ ZSTD_OBJS = \ $O\xxhash.obj \ + $O\entropy_common.obj \ + $O\fse_decompress.obj \ + $O\fse_compress.obj \ + $O\huf_compress.obj \ + $O\huf_decompress.obj \ + $O\error_private.obj \ ZSTDMT_OBJS = \ $O\threading.obj \ diff --git a/CPP/7zip/Bundles/Codec_lizard/resource.rc b/CPP/7zip/Bundles/Codec_lizard/resource.rc index c85cfa1d..79afb0bf 100644 --- a/CPP/7zip/Bundles/Codec_lizard/resource.rc +++ b/CPP/7zip/Bundles/Codec_lizard/resource.rc @@ -1,6 +1,6 @@ #include "../../../../C/7zVersionTr.h" #include "../../../../C/7zVersion.rc" -MY_VERSION_INFO_DLL("7-Zip Lizard Plugin v2.0", "Lizard") +MY_VERSION_INFO_DLL("7-Zip Lizard Plugin v1.0", "Lizard") -101 ICON "../../Archive/Icons/7z.ico" +101 ICON "../../Archive/Icons/liz.ico" diff --git a/CPP/7zip/Bundles/Codec_lz4/resource.rc b/CPP/7zip/Bundles/Codec_lz4/resource.rc index 00e9c490..9f53aae2 100644 --- a/CPP/7zip/Bundles/Codec_lz4/resource.rc +++ b/CPP/7zip/Bundles/Codec_lz4/resource.rc @@ -3,4 +3,4 @@ MY_VERSION_INFO_DLL("7-Zip LZ4 Plugin v1.7.5", "Lz4") -101 ICON "../../Archive/Icons/7z.ico" +101 ICON "../../Archive/Icons/lz5.ico" diff --git a/CPP/7zip/Bundles/Codec_lz5/resource.rc b/CPP/7zip/Bundles/Codec_lz5/resource.rc index 15aa9f1a..27467f1e 100644 --- a/CPP/7zip/Bundles/Codec_lz5/resource.rc +++ b/CPP/7zip/Bundles/Codec_lz5/resource.rc @@ -3,4 +3,4 @@ MY_VERSION_INFO_DLL("7-Zip LZ5 Plugin v1.5", "Lz5") -101 ICON "../../Archive/Icons/7z.ico" +101 ICON "../../Archive/Icons/lz5.ico" diff --git a/CPP/7zip/Bundles/Codec_zstd/resource.rc b/CPP/7zip/Bundles/Codec_zstd/resource.rc index 4290d7e0..3c2421d0 100644 --- a/CPP/7zip/Bundles/Codec_zstd/resource.rc +++ b/CPP/7zip/Bundles/Codec_zstd/resource.rc @@ -3,4 +3,4 @@ MY_VERSION_INFO_DLL("7-Zip ZStandard Plugin v1.2.0", "zstd") -101 ICON "../../Archive/Icons/7z.ico" +101 ICON "../../Archive/Icons/zstd.ico" diff --git a/CPP/7zip/Bundles/Codec_zstdF/resource.rc b/CPP/7zip/Bundles/Codec_zstdF/resource.rc index 4290d7e0..3c2421d0 100644 --- a/CPP/7zip/Bundles/Codec_zstdF/resource.rc +++ b/CPP/7zip/Bundles/Codec_zstdF/resource.rc @@ -3,4 +3,4 @@ MY_VERSION_INFO_DLL("7-Zip ZStandard Plugin v1.2.0", "zstd") -101 ICON "../../Archive/Icons/7z.ico" +101 ICON "../../Archive/Icons/zstd.ico" diff --git a/CPP/7zip/Bundles/Format7z/makefile b/CPP/7zip/Bundles/Format7z/makefile index 93b1174e..2d1ca574 100644 --- a/CPP/7zip/Bundles/Format7z/makefile +++ b/CPP/7zip/Bundles/Format7z/makefile @@ -178,14 +178,9 @@ BROTLI_OBJS = \ $O/br_utf8_util.obj \ LIZARD_OBJS = \ - $O/liz_entropy_common.obj \ - $O/liz_fse_compress.obj \ - $O/liz_fse_decompress.obj \ - $O/liz_huf_compress.obj \ - $O/liz_huf_decompress.obj \ - $O/liz_lz5_compress.obj \ - $O/liz_lz5_decompress.obj \ - $O/liz_lz5frame.obj \ + $O/lizard_compress.obj \ + $O/lizard_decompress.obj \ + $O/lizard_frame.obj \ LZ4_OBJS = \ $O\lz4.obj \ diff --git a/CPP/7zip/Bundles/Format7zF/Arc.mak b/CPP/7zip/Bundles/Format7zF/Arc.mak index 58a53fa6..5e7ba6f8 100644 --- a/CPP/7zip/Bundles/Format7zF/Arc.mak +++ b/CPP/7zip/Bundles/Format7zF/Arc.mak @@ -73,6 +73,7 @@ AR_OBJS = \ $O\LzHandler.obj \ $O\Lz4Handler.obj \ $O\Lz5Handler.obj \ + $O\LizardHandler.obj \ $O\LzhHandler.obj \ $O\LzmaHandler.obj \ $O\MachoHandler.obj \ diff --git a/CPP/7zip/Bundles/Format7zF/makefile b/CPP/7zip/Bundles/Format7zF/makefile index 206f236a..17f36587 100644 --- a/CPP/7zip/Bundles/Format7zF/makefile +++ b/CPP/7zip/Bundles/Format7zF/makefile @@ -58,14 +58,9 @@ BROTLI_OBJS = \ $O/br_utf8_util.obj \ LIZARD_OBJS = \ - $O/liz_entropy_common.obj \ - $O/liz_fse_compress.obj \ - $O/liz_fse_decompress.obj \ - $O/liz_huf_compress.obj \ - $O/liz_huf_decompress.obj \ - $O/liz_lz5_compress.obj \ - $O/liz_lz5_decompress.obj \ - $O/liz_lz5frame.obj \ + $O/lizard_compress.obj \ + $O/lizard_decompress.obj \ + $O/lizard_frame.obj \ LZ4_OBJS = \ $O\lz4.obj \ diff --git a/CPP/7zip/Bundles/Format7zF/resource.rc b/CPP/7zip/Bundles/Format7zF/resource.rc index 52489c64..e7c4d1b2 100644 --- a/CPP/7zip/Bundles/Format7zF/resource.rc +++ b/CPP/7zip/Bundles/Format7zF/resource.rc @@ -31,9 +31,10 @@ MY_VERSION_INFO_DLL("7z Plugin" , "7z") 25 ICON "../../Archive/Icons/zst.ico" 26 ICON "../../Archive/Icons/lz4.ico" 27 ICON "../../Archive/Icons/lz5.ico" +28 ICON "../../Archive/Icons/liz.ico" STRINGTABLE BEGIN - 100 "7z:0 zip:1 rar:3 001:9 cab:7 iso:8 xz:23 txz:23 lzma:16 tar:13 cpio:12 bz2:2 bzip2:2 tbz2:2 tbz:2 gz:14 gzip:14 tgz:14 tpz:14 z:5 taz:5 lz:16 tlz:16 lz4:26 lz5:27 lzh:6 lha:6 rpm:10 deb:11 arj:4 vhd:20 wim:15 swm:15 fat:21 ntfs:22 dmg:17 hfs:18 xar:19 squashfs:24 zst:25" + 100 "7z:0 zip:1 rar:3 001:9 cab:7 iso:8 xz:23 txz:23 lzma:16 tar:13 cpio:12 bz2:2 bzip2:2 tbz2:2 tbz:2 gz:14 gzip:14 tgz:14 tpz:14 z:5 taz:5 lz:16 tlz:16 liz:28 lz4:26 lz5:27 lzh:6 lha:6 rpm:10 deb:11 arj:4 vhd:20 wim:15 swm:15 fat:21 ntfs:22 dmg:17 hfs:18 xar:19 squashfs:24 zst:25" END diff --git a/CPP/7zip/Compress/LizardDecoder.h b/CPP/7zip/Compress/LizardDecoder.h index 5be5e401..2fbba01c 100644 --- a/CPP/7zip/Compress/LizardDecoder.h +++ b/CPP/7zip/Compress/LizardDecoder.h @@ -3,8 +3,9 @@ #define LIZARD_STATIC_LINKING_ONLY #include "../../../C/Alloc.h" #include "../../../C/Threads.h" -#include "../../../C/lizard/liz_compress.h" -#include "../../../C/lizard/lizframe.h" +#include "../../../C/lizard/lizard_compress.h" +#include "../../../C/lizard/lizard_decompress.h" +#include "../../../C/lizard/lizard_frame.h" #include "../../../C/zstdmt/lizard-mt.h" #include "../../Windows/System.h" @@ -37,8 +38,8 @@ struct DProps void clear () { memset(this, 0, sizeof (*this)); - _ver_major = LIZ_VERSION_MAJOR; - _ver_minor = LIZ_VERSION_MINOR; + _ver_major = LIZARD_VERSION_MAJOR; + _ver_minor = LIZARD_VERSION_MINOR; _level = 1; } diff --git a/CPP/7zip/Compress/LizardEncoder.h b/CPP/7zip/Compress/LizardEncoder.h index 3beb520f..3b6126b4 100644 --- a/CPP/7zip/Compress/LizardEncoder.h +++ b/CPP/7zip/Compress/LizardEncoder.h @@ -3,8 +3,8 @@ #define LIZARD_STATIC_LINKING_ONLY #include "../../../C/Alloc.h" #include "../../../C/Threads.h" -#include "../../../C/lizard/liz_compress.h" -#include "../../../C/lizard/lizframe.h" +#include "../../../C/lizard/lizard_compress.h" +#include "../../../C/lizard/lizard_frame.h" #include "../../../C/zstdmt/lizard-mt.h" #include "../../Common/Common.h" @@ -22,8 +22,8 @@ struct CProps void clear () { memset(this, 0, sizeof (*this)); - _ver_major = LIZ_VERSION_MAJOR; - _ver_minor = LIZ_VERSION_MINOR; + _ver_major = LIZARD_VERSION_MAJOR; + _ver_minor = LIZARD_VERSION_MINOR; _level = 3; } diff --git a/CPP/7zip/Guid.txt b/CPP/7zip/Guid.txt index f91fbfc8..6314bb34 100644 --- a/CPP/7zip/Guid.txt +++ b/CPP/7zip/Guid.txt @@ -166,6 +166,7 @@ Handler GUIDs: 0E zstd 0F lz4 10 lz5 + 11 liz C6 Lzip C7 Ext diff --git a/CPP/7zip/UI/GUI/CompressDialog.cpp b/CPP/7zip/UI/GUI/CompressDialog.cpp index f6f67e6e..31052193 100644 --- a/CPP/7zip/UI/GUI/CompressDialog.cpp +++ b/CPP/7zip/UI/GUI/CompressDialog.cpp @@ -99,9 +99,12 @@ enum EMethodID kCopy, kZSTD, kBROTLI, - kLIZARD, kLZ4, kLZ5, + kLIZARD_M1, + kLIZARD_M2, + kLIZARD_M3, + kLIZARD_M4, kLZMA, kLZMA2, kPPMd, @@ -111,14 +114,37 @@ enum EMethodID kPPMdZip }; +static LPCSTR const kMethodsLongnames[] = +{ + "Copy" + , "ZStandard" + , "Brotli" + , "LZ4" + , "LZ5" + , "Lizard, FastLZ4" + , "Lizard, LIZv1" + , "Lizard, FastLZ4 + Huffman" + , "Lizard, LIZv1 + Huffman" + , "LZMA" + , "LZMA2" + , "PPMd" + , "BZip2" + , "Deflate" + , "Deflate64" + , "PPMd" +}; + static LPCSTR const kMethodsNames[] = { "Copy" - , "ZSTD" - , "BROTLI" - , "LIZARD" + , "zstd" + , "Brotli" , "LZ4" , "LZ5" + , "Lizard" + , "Lizard" + , "Lizard" + , "Lizard" , "LZMA" , "LZMA2" , "PPMd" @@ -140,7 +166,10 @@ static const EMethodID g_BrotliMethods[] = static const EMethodID g_LizardMethods[] = { - kLIZARD + kLIZARD_M1, + kLIZARD_M2, + kLIZARD_M3, + kLIZARD_M4 }; static const EMethodID g_Lz4Methods[] = @@ -157,9 +186,12 @@ static const EMethodID g_7zMethods[] = { kZSTD, kBROTLI, - kLIZARD, kLZ4, kLZ5, + kLIZARD_M1, + kLIZARD_M2, + kLIZARD_M3, + kLIZARD_M4, kLZMA2, kLZMA, kPPMd, @@ -269,25 +301,25 @@ static const CFormatInfo g_Formats[] = false, false, true, false, false, false }, { - "Brotli", + "Brotli", /* 7 */ (1 << 0) | (1 << 1) | (1 << 3) | (1 << 6) | (1 << 9) | (1 << 11), METHODS_PAIR(g_BrotliMethods), false, false, true, false, false, false }, { - "Lizard", /* 10..19 / 20..29 / .... */ + "Lizard", /* 8 */ (1 << 10) | (1 << 11) | (1 << 13) | (1 << 15) | (1 << 17) | (1 << 19), METHODS_PAIR(g_LizardMethods), false, false, true, false, false, false }, { - "LZ4", + "LZ4", /* 9 */ (1 << 0) | (1 << 1) | (1 << 3) | (1 << 6) | (1 << 9) | (1 << 12), METHODS_PAIR(g_Lz4Methods), false, false, true, false, false, false }, { - "LZ5", + "LZ5", /* 10 */ (1 << 0) | (1 << 1) | (1 << 3) | (1 << 7) | (1 << 11) | (1 << 15), METHODS_PAIR(g_Lz5Methods), false, false, true, false, false, false @@ -1082,6 +1114,7 @@ void CCompressDialog::SetLevel() UInt32 level = GetLevel2(); UInt32 LevelsMask; UInt32 LevelsStart = 0; + UInt32 LevelsEnd = 22; UInt32 langID = 0; unsigned i, ir; @@ -1104,9 +1137,22 @@ void CCompressDialog::SetLevel() LevelsMask = g_Formats[6].LevelsMask; else if (GetMethodID() == kBROTLI) LevelsMask = g_Formats[7].LevelsMask; - else if (GetMethodID() == kLIZARD) { + else if (GetMethodID() == kLIZARD_M1) { LevelsMask = g_Formats[8].LevelsMask; LevelsStart = 10; + LevelsEnd = 19; + } else if (GetMethodID() == kLIZARD_M2) { + LevelsMask = g_Formats[8].LevelsMask; + LevelsStart = 20; + LevelsEnd = 29; + } else if (GetMethodID() == kLIZARD_M3) { + LevelsMask = g_Formats[8].LevelsMask; + LevelsStart = 30; + LevelsEnd = 39; + } else if (GetMethodID() == kLIZARD_M4) { + LevelsMask = g_Formats[8].LevelsMask; + LevelsStart = 40; + LevelsEnd = 49; } else if (GetMethodID() == kLZ4) LevelsMask = g_Formats[9].LevelsMask; else if (GetMethodID() == kLZ5) @@ -1114,13 +1160,13 @@ void CCompressDialog::SetLevel() else LevelsMask = g_Formats[GetStaticFormatIndex()].LevelsMask; - for (i = LevelsStart; i <= 49; i++) + for (i = LevelsStart; i <= LevelsEnd; i++) { TCHAR s[40]; TCHAR t[50] = { TEXT('L'), TEXT('e'), TEXT('v'), TEXT('e'), TEXT('l'), TEXT(' '), 0 }; // lizard needs extra handling - if (GetMethodID() == kLIZARD) { + if (GetMethodID() >= kLIZARD_M1 && GetMethodID() <= kLIZARD_M4) { ir = i; if (ir % 10 == 0) langID = 0; while (ir > 19) { ir -= 10; } @@ -1153,7 +1199,6 @@ void CCompressDialog::SetLevel() return; } - static LRESULT ComboBox_AddStringAscii(NControl::CComboBox &cb, const char *s) { return cb.AddString((CSysString)s); @@ -1174,33 +1219,56 @@ void CCompressDialog::SetMethod(int keepMethodId) const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; int index = FindRegistryFormat(ai.Name); UString defaultMethod; - if (index >= 0) - { + int defaultLevel = 5; + if (index >= 0) { const NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index]; defaultMethod = fo.Method; + defaultLevel = fo.Level; } + bool isSfx = IsSFX(); bool weUseSameMethod = false; - + +#if 0 + { + wchar_t buf[2000]; + wsprintf(buf, L"keep=%d level = %d method=%s", keepMethodId, defaultLevel, (LPCWSTR)defaultMethod); + ShowErrorMessage(*this, buf); + } +#endif + + for (unsigned m = 0; m < fi.NumMethods; m++) { EMethodID methodID = fi.MathodIDs[m]; if (isSfx) if (!IsMethodSupportedBySfx(methodID)) continue; + + const char *methodLong = kMethodsLongnames[methodID]; const char *method = kMethodsNames[methodID]; - int itemIndex = (int)ComboBox_AddStringAscii(m_Method, method); + int itemIndex = (int)ComboBox_AddStringAscii(m_Method, methodLong); m_Method.SetItemData(itemIndex, methodID); - if (keepMethodId == methodID) - { - m_Method.SetCurSel(itemIndex); + + if (keepMethodId == methodID) { weUseSameMethod = true; + m_Method.SetCurSel(itemIndex); continue; } - if ((defaultMethod.IsEqualTo_Ascii_NoCase(method) || m == 0) && !weUseSameMethod) + + // Lizard :/ + if (defaultMethod.IsEqualTo_Ascii_NoCase("lizard") && keepMethodId == -1) { + if (defaultLevel >= 10 && defaultLevel <= 19) m_Method.SetCurSel(kLIZARD_M1 - 1); + if (defaultLevel >= 20 && defaultLevel <= 29) m_Method.SetCurSel(kLIZARD_M2 - 1); + if (defaultLevel >= 30 && defaultLevel <= 39) m_Method.SetCurSel(kLIZARD_M3 - 1); + if (defaultLevel >= 40 && defaultLevel <= 49) m_Method.SetCurSel(kLIZARD_M4 - 1); + } + + if ((defaultMethod.IsEqualTo_Ascii_NoCase(method) || m == 0) && !weUseSameMethod) { m_Method.SetCurSel(itemIndex); + } } - + if (!weUseSameMethod) { SetDictionary(); @@ -1672,7 +1740,10 @@ void CCompressDialog::SetNumThreads() case kBROTLI: numAlgoThreadsMax = 128; break; case kLZ4: numAlgoThreadsMax = 128; break; case kLZ5: numAlgoThreadsMax = 128; break; - case kLIZARD: numAlgoThreadsMax = 128; break; + case kLIZARD_M1: numAlgoThreadsMax = 128; break; + case kLIZARD_M2: numAlgoThreadsMax = 128; break; + case kLIZARD_M3: numAlgoThreadsMax = 128; break; + case kLIZARD_M4: numAlgoThreadsMax = 128; break; case kLZMA: numAlgoThreadsMax = 2; break; case kLZMA2: numAlgoThreadsMax = 32; break; case kBZip2: numAlgoThreadsMax = 32; break; diff --git a/DOC/Methods-Extern.md b/DOC/Methods-Extern.md index 698b2cc4..d31e4a46 100644 --- a/DOC/Methods-Extern.md +++ b/DOC/Methods-Extern.md @@ -62,9 +62,9 @@ License: Zstandard library is provided as open source software using the BSD license. 7-Zip Container Header: -This header is mandatory and must be exact 5 bytes. The data within that header -is for informational purposes only and not used by the decoder. If the header -is not there, or has another size, the decoder will not decompress the content. + +This header is for informational purposes only. It's not mandatory for +decompressing the data. ``` C Byte _ver_major; // currently 1 Byte _ver_minor; // currently 2 @@ -119,9 +119,9 @@ License: The Brotli library is provided as open source software using the MIT license. 7-Zip Container Header (3 bytes): -This header is mandatory and must be exact 3 bytes. The data within that header -is for informational purposes only and not used by the decoder. If the header -is not there, or has another size, the decoder will not decompress the content. + +This header is for informational purposes only. It's not mandatory for +decompressing the data. ``` C Byte _ver_major; // currently 0 Byte _ver_minor; // currently 6 @@ -164,9 +164,9 @@ License: LZ4 library is provided as open source software using the BSD license. 7-Zip Container Header: -This header is mandatory and must be exact 5 bytes. The data within that header -is for informational purposes only and not used by the decoder. If the header -is not there, or has another size, the decoder will not decompress the content. + +This header is for informational purposes only. It's not mandatory for +decompressing the data. ``` C Byte _ver_major; // currently 1 Byte _ver_minor; // currently 7 @@ -215,9 +215,9 @@ License: LZ5 library is provided as open source software using the BSD license. 7-Zip Container Header: -This header is mandatory and must be exact 5 bytes. The data within that header -is for informational purposes only and not used by the decoder. If the header -is not there, or has another size, the decoder will not decompress the content. + +This header is for informational purposes only. It's not mandatory for +decompressing the data. ``` C Byte _ver_major; // currently 1 Byte _ver_minor; // currently 5 @@ -264,9 +264,9 @@ License: Lizard library is provided as open source software using the BSD license. 7-Zip Container Header: -This header is mandatory and must be exact 3 bytes. The data within that header -is for informational purposes only and not used by the decoder. If the header -is not there, or has another size, the decoder will not decompress the content. + +This header is for informational purposes only. It's not mandatory for +decompressing the data. ``` C Byte _ver_major; // currently 2 Byte _ver_minor; // currently 0