diff options
Diffstat (limited to 'doc/zstd_manual.html')
| -rw-r--r-- | doc/zstd_manual.html | 929 |
1 files changed, 530 insertions, 399 deletions
diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index 6dfa6d997cb90..f1628b50d08d3 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -1,37 +1,46 @@ <html> <head> <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"> -<title>zstd 1.3.8 Manual</title> +<title>zstd 1.4.0 Manual</title> </head> <body> -<h1>zstd 1.3.8 Manual</h1> +<h1>zstd 1.4.0 Manual</h1> <hr> <a name="Contents"></a><h2>Contents</h2> <ol> <li><a href="#Chapter1">Introduction</a></li> <li><a href="#Chapter2">Version</a></li> <li><a href="#Chapter3">Default constant</a></li> -<li><a href="#Chapter4">Simple API</a></li> -<li><a href="#Chapter5">Explicit context</a></li> -<li><a href="#Chapter6">Simple dictionary API</a></li> -<li><a href="#Chapter7">Bulk processing dictionary API</a></li> -<li><a href="#Chapter8">Streaming</a></li> -<li><a href="#Chapter9">Streaming compression - HowTo</a></li> -<li><a href="#Chapter10">Streaming decompression - HowTo</a></li> -<li><a href="#Chapter11">ADVANCED AND EXPERIMENTAL FUNCTIONS</a></li> -<li><a href="#Chapter12">Candidate API for promotion to stable status</a></li> -<li><a href="#Chapter13">Advanced compression API</a></li> -<li><a href="#Chapter14">experimental API (static linking only)</a></li> -<li><a href="#Chapter15">Frame size functions</a></li> -<li><a href="#Chapter16">Memory management</a></li> -<li><a href="#Chapter17">Advanced compression functions</a></li> -<li><a href="#Chapter18">Advanced decompression functions</a></li> -<li><a href="#Chapter19">Advanced streaming functions</a></li> -<li><a href="#Chapter20">Buffer-less and synchronous inner streaming functions</a></li> -<li><a href="#Chapter21">Buffer-less streaming compression (synchronous mode)</a></li> -<li><a href="#Chapter22">Buffer-less streaming decompression (synchronous mode)</a></li> -<li><a href="#Chapter23">ZSTD_getFrameHeader() :</a></li> -<li><a href="#Chapter24">Block level API</a></li> +<li><a href="#Chapter4">Constants</a></li> +<li><a href="#Chapter5">Simple API</a></li> +<li><a href="#Chapter6">Explicit context</a></li> +<li><a href="#Chapter7">Advanced compression API</a></li> +<li><a href="#Chapter8">Advanced decompression API</a></li> +<li><a href="#Chapter9">Streaming</a></li> +<li><a href="#Chapter10">Streaming compression - HowTo</a></li> +<li><a href="#Chapter11">This is a legacy streaming API, and can be replaced by ZSTD_CCtx_reset() and</a></li> +<li><a href="#Chapter12">Equivalent to:</a></li> +<li><a href="#Chapter13">Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).</a></li> +<li><a href="#Chapter14">Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush).</a></li> +<li><a href="#Chapter15">Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end).</a></li> +<li><a href="#Chapter16">Streaming decompression - HowTo</a></li> +<li><a href="#Chapter17">Simple dictionary API</a></li> +<li><a href="#Chapter18">Bulk processing dictionary API</a></li> +<li><a href="#Chapter19">Dictionary helper functions</a></li> +<li><a href="#Chapter20">Advanced dictionary and prefix API</a></li> +<li><a href="#Chapter21">ADVANCED AND EXPERIMENTAL FUNCTIONS</a></li> +<li><a href="#Chapter22">experimental API (static linking only)</a></li> +<li><a href="#Chapter23">Frame size functions</a></li> +<li><a href="#Chapter24">ZSTD_decompressBound() :</a></li> +<li><a href="#Chapter25">Memory management</a></li> +<li><a href="#Chapter26">Advanced compression functions</a></li> +<li><a href="#Chapter27">Advanced decompression functions</a></li> +<li><a href="#Chapter28">Advanced streaming functions</a></li> +<li><a href="#Chapter29">Buffer-less and synchronous inner streaming functions</a></li> +<li><a href="#Chapter30">Buffer-less streaming compression (synchronous mode)</a></li> +<li><a href="#Chapter31">Buffer-less streaming decompression (synchronous mode)</a></li> +<li><a href="#Chapter32">ZSTD_getFrameHeader() :</a></li> +<li><a href="#Chapter33">Block level API</a></li> </ol> <hr> <a name="Chapter1"></a><h2>Introduction</h2><pre> @@ -71,7 +80,9 @@ </b></pre><BR> <a name="Chapter3"></a><h2>Default constant</h2><pre></pre> -<a name="Chapter4"></a><h2>Simple API</h2><pre></pre> +<a name="Chapter4"></a><h2>Constants</h2><pre></pre> + +<a name="Chapter5"></a><h2>Simple API</h2><pre></pre> <pre><b>size_t ZSTD_compress( void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -126,13 +137,22 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. </p></pre><BR> +<pre><b>size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); +</b><p> `src` should point to the start of a ZSTD frame or skippable frame. + `srcSize` must be >= first frame size + @return : the compressed size of the first frame starting at `src`, + suitable to pass as `srcSize` to `ZSTD_decompress` or similar, + or an error code if input is invalid +</p></pre><BR> + <h3>Helper functions</h3><pre></pre><b><pre>#define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) </b>/* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */<b> size_t ZSTD_compressBound(size_t srcSize); </b>/*!< maximum compressed size in worst case single-pass scenario */<b> unsigned ZSTD_isError(size_t code); </b>/*!< tells if a `size_t` function result is an error code */<b> const char* ZSTD_getErrorName(size_t code); </b>/*!< provides readable string from an error code */<b> +int ZSTD_minCLevel(void); </b>/*!< minimum negative compression level allowed */<b> int ZSTD_maxCLevel(void); </b>/*!< maximum compression level available */<b> </pre></b><BR> -<a name="Chapter5"></a><h2>Explicit context</h2><pre></pre> +<a name="Chapter6"></a><h2>Explicit context</h2><pre></pre> <h3>Compression context</h3><pre> When compressing many times, it is recommended to allocate a context just once, and re-use it for each successive compression operation. @@ -169,228 +189,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); </p></pre><BR> -<a name="Chapter6"></a><h2>Simple dictionary API</h2><pre></pre> - -<pre><b>size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize, - int compressionLevel); -</b><p> Compression at an explicit compression level using a Dictionary. - A dictionary can be any arbitrary data segment (also called a prefix), - or a buffer with specified information (see dictBuilder/zdict.h). - Note : This function loads the dictionary, resulting in significant startup delay. - It's intended for a dictionary used only once. - Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. -</p></pre><BR> - -<pre><b>size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize); -</b><p> Decompression using a known Dictionary. - Dictionary must be identical to the one used during compression. - Note : This function loads the dictionary, resulting in significant startup delay. - It's intended for a dictionary used only once. - Note : When `dict == NULL || dictSize < 8` no dictionary is used. -</p></pre><BR> - -<a name="Chapter7"></a><h2>Bulk processing dictionary API</h2><pre></pre> - -<pre><b>ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, - int compressionLevel); -</b><p> When compressing multiple messages / blocks using the same dictionary, it's recommended to load it only once. - ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup cost. - ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. - `dictBuffer` can be released after ZSTD_CDict creation, because its content is copied within CDict. - Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate `dictBuffer` content. - Note : A ZSTD_CDict can be created from an empty dictBuffer, but it is inefficient when used to compress small data. -</p></pre><BR> - -<pre><b>size_t ZSTD_freeCDict(ZSTD_CDict* CDict); -</b><p> Function frees memory allocated by ZSTD_createCDict(). -</p></pre><BR> - -<pre><b>size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_CDict* cdict); -</b><p> Compression using a digested Dictionary. - Recommended when same dictionary is used multiple times. - Note : compression level is _decided at dictionary creation time_, - and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) -</p></pre><BR> - -<pre><b>ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); -</b><p> Create a digested dictionary, ready to start decompression operation without startup delay. - dictBuffer can be released after DDict creation, as its content is copied inside DDict. -</p></pre><BR> - -<pre><b>size_t ZSTD_freeDDict(ZSTD_DDict* ddict); -</b><p> Function frees memory allocated with ZSTD_createDDict() -</p></pre><BR> - -<pre><b>size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_DDict* ddict); -</b><p> Decompression using a digested Dictionary. - Recommended when same dictionary is used multiple times. -</p></pre><BR> - -<a name="Chapter8"></a><h2>Streaming</h2><pre></pre> - -<pre><b>typedef struct ZSTD_inBuffer_s { - const void* src; </b>/**< start of input buffer */<b> - size_t size; </b>/**< size of input buffer */<b> - size_t pos; </b>/**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */<b> -} ZSTD_inBuffer; -</b></pre><BR> -<pre><b>typedef struct ZSTD_outBuffer_s { - void* dst; </b>/**< start of output buffer */<b> - size_t size; </b>/**< size of output buffer */<b> - size_t pos; </b>/**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */<b> -} ZSTD_outBuffer; -</b></pre><BR> -<a name="Chapter9"></a><h2>Streaming compression - HowTo</h2><pre> - A ZSTD_CStream object is required to track streaming operation. - Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. - ZSTD_CStream objects can be reused multiple times on consecutive compression operations. - It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory. - - For parallel execution, use one separate ZSTD_CStream per thread. - - note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing. - - Parameters are sticky : when starting a new compression on the same context, - it will re-use the same sticky parameters as previous compression session. - When in doubt, it's recommended to fully initialize the context before usage. - Use ZSTD_initCStream() to set the parameter to a selected compression level. - Use advanced API (ZSTD_CCtx_setParameter(), etc.) to set more specific parameters. - - Use ZSTD_compressStream() as many times as necessary to consume input stream. - The function will automatically update both `pos` fields within `input` and `output`. - Note that the function may not consume the entire input, - for example, because the output buffer is already full, - in which case `input.pos < input.size`. - The caller must check if input has been entirely consumed. - If not, the caller must make some room to receive more compressed data, - and then present again remaining input data. - @return : a size hint, preferred nb of bytes to use as input for next function call - or an error code, which can be tested using ZSTD_isError(). - Note 1 : it's just a hint, to help latency a little, any value will work fine. - Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize() - - At any moment, it's possible to flush whatever data might remain stuck within internal buffer, - using ZSTD_flushStream(). `output->pos` will be updated. - Note that, if `output->size` is too small, a single invocation of ZSTD_flushStream() might not be enough (return code > 0). - In which case, make some room to receive more compressed data, and call again ZSTD_flushStream(). - @return : 0 if internal buffers are entirely flushed, - >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), - or an error code, which can be tested using ZSTD_isError(). - - ZSTD_endStream() instructs to finish a frame. - It will perform a flush and write frame epilogue. - The epilogue is required for decoders to consider a frame completed. - flush() operation is the same, and follows same rules as ZSTD_flushStream(). - @return : 0 if frame fully completed and fully flushed, - >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), - or an error code, which can be tested using ZSTD_isError(). - - -<BR></pre> - -<pre><b>typedef ZSTD_CCtx ZSTD_CStream; </b>/**< CCtx and CStream are now effectively same object (>= v1.3.0) */<b> -</b></pre><BR> -<h3>ZSTD_CStream management functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream(void); -size_t ZSTD_freeCStream(ZSTD_CStream* zcs); -</pre></b><BR> -<h3>Streaming compression functions</h3><pre></pre><b><pre>size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); -size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); -size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); -size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); -</pre></b><BR> -<pre><b>size_t ZSTD_CStreamInSize(void); </b>/**< recommended size for input buffer */<b> -</b></pre><BR> -<pre><b>size_t ZSTD_CStreamOutSize(void); </b>/**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */<b> -</b></pre><BR> -<a name="Chapter10"></a><h2>Streaming decompression - HowTo</h2><pre> - A ZSTD_DStream object is required to track streaming operations. - Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. - ZSTD_DStream objects can be re-used multiple times. - - Use ZSTD_initDStream() to start a new decompression operation. - @return : recommended first input size - Alternatively, use advanced API to set specific properties. - - Use ZSTD_decompressStream() repetitively to consume your input. - The function will update both `pos` fields. - If `input.pos < input.size`, some input has not been consumed. - It's up to the caller to present again remaining data. - The function tries to flush all data decoded immediately, respecting output buffer size. - If `output.pos < output.size`, decoder has flushed everything it could. - But if `output.pos == output.size`, there might be some data left within internal buffers., - In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer. - Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX. - @return : 0 when a frame is completely decoded and fully flushed, - or an error code, which can be tested using ZSTD_isError(), - or any other value > 0, which means there is still some decoding or flushing to do to complete current frame : - the return value is a suggested next input size (just a hint for better latency) - that will never request more than the remaining frame size. - -<BR></pre> - -<pre><b>typedef ZSTD_DCtx ZSTD_DStream; </b>/**< DCtx and DStream are now effectively same object (>= v1.3.0) */<b> -</b></pre><BR> -<h3>ZSTD_DStream management functions</h3><pre></pre><b><pre>ZSTD_DStream* ZSTD_createDStream(void); -size_t ZSTD_freeDStream(ZSTD_DStream* zds); -</pre></b><BR> -<h3>Streaming decompression functions</h3><pre></pre><b><pre>size_t ZSTD_initDStream(ZSTD_DStream* zds); -size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); -</pre></b><BR> -<pre><b>size_t ZSTD_DStreamInSize(void); </b>/*!< recommended size for input buffer */<b> -</b></pre><BR> -<pre><b>size_t ZSTD_DStreamOutSize(void); </b>/*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */<b> -</b></pre><BR> -<a name="Chapter11"></a><h2>ADVANCED AND EXPERIMENTAL FUNCTIONS</h2><pre> - The definitions in the following section are considered experimental. - They are provided for advanced scenarios. - They should never be used with a dynamic library, as prototypes may change in the future. - Use them only in association with static linking. - -<BR></pre> - -<a name="Chapter12"></a><h2>Candidate API for promotion to stable status</h2><pre> - The following symbols and constants form the "staging area" : - they are considered to join "stable API" by v1.4.0. - The proposal is written so that it can be made stable "as is", - though it's still possible to suggest improvements. - Staging is in fact last chance for changes, - the API is locked once reaching "stable" status. - -<BR></pre> - -<pre><b>int ZSTD_minCLevel(void); </b>/*!< minimum negative compression level allowed */<b> -</b></pre><BR> -<pre><b>size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); -</b><p> `src` should point to the start of a ZSTD frame or skippable frame. - `srcSize` must be >= first frame size - @return : the compressed size of the first frame starting at `src`, - suitable to pass as `srcSize` to `ZSTD_decompress` or similar, - or an error code if input is invalid -</p></pre><BR> - -<pre><b>size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); -size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); -size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); -size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); -size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); -size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); -</b><p> These functions give the _current_ memory usage of selected object. - Note that object memory usage can evolve (increase or decrease) over time. -</p></pre><BR> - -<a name="Chapter13"></a><h2>Advanced compression API</h2><pre></pre> +<a name="Chapter7"></a><h2>Advanced compression API</h2><pre></pre> <pre><b>typedef enum { ZSTD_fast=1, ZSTD_dfast=2, @@ -407,7 +206,10 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); </b></pre><BR> <pre><b>typedef enum { - </b>/* compression parameters */<b> + </b>/* compression parameters<b> + * Note: When compressing with a ZSTD_CDict these parameters are superseded + * by the parameters used to construct the ZSTD_CDict. See ZSTD_CCtx_refCDict() + * for more info (superseded-by-cdict). */ ZSTD_c_compressionLevel=100, </b>/* Update all compression parameters according to pre-defined cLevel table<b> * Default level is ZSTD_CLEVEL_DEFAULT==3. * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. @@ -529,6 +331,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); * ZSTD_c_format * ZSTD_c_forceMaxWindow * ZSTD_c_forceAttachDict + * ZSTD_c_literalCompressionMode * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * note : never ever use experimentalParam? names directly; * also, the enums values themselves are unstable and can still change. @@ -536,7 +339,8 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); ZSTD_c_experimentalParam1=500, ZSTD_c_experimentalParam2=10, ZSTD_c_experimentalParam3=1000, - ZSTD_c_experimentalParam4=1001 + ZSTD_c_experimentalParam4=1001, + ZSTD_c_experimentalParam5=1002, } ZSTD_cParameter; </b></pre><BR> <pre><b>typedef struct { @@ -580,62 +384,10 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); Note 3 : Whenever all input data is provided and consumed in a single round, for example with ZSTD_compress2(), or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end), - this value is automatically overriden by srcSize instead. + this value is automatically overridden by srcSize instead. </p></pre><BR> -<pre><b>size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); -</b><p> Create an internal CDict from `dict` buffer. - Decompression will have to use same dictionary. - @result : 0, or an error code (which can be tested with ZSTD_isError()). - Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, - meaning "return to no-dictionary mode". - Note 1 : Dictionary is sticky, it will be used for all future compressed frames. - To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters). - Note 2 : Loading a dictionary involves building tables. - It's also a CPU consuming operation, with non-negligible impact on latency. - Tables are dependent on compression parameters, and for this reason, - compression parameters can no longer be changed after loading a dictionary. - Note 3 :`dict` content will be copied internally. - Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead. - In such a case, dictionary buffer must outlive its users. - Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() - to precisely select how dictionary content must be interpreted. -</p></pre><BR> - -<pre><b>size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); -</b><p> Reference a prepared dictionary, to be used for all next compressed frames. - Note that compression parameters are enforced from within CDict, - and supercede any compression parameter previously set within CCtx. - The dictionary will remain valid for future compressed frames using same CCtx. - @result : 0, or an error code (which can be tested with ZSTD_isError()). - Special : Referencing a NULL CDict means "return to no-dictionary mode". - Note 1 : Currently, only one dictionary can be managed. - Referencing a new dictionary effectively "discards" any previous one. - Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. -</p></pre><BR> - -<pre><b>size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, - const void* prefix, size_t prefixSize); -</b><p> Reference a prefix (single-usage dictionary) for next compressed frame. - A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end). - Decompression will need same prefix to properly regenerate data. - Compressing with a prefix is similar in outcome as performing a diff and compressing it, - but performs much faster, especially during decompression (compression speed is tunable with compression level). - @result : 0, or an error code (which can be tested with ZSTD_isError()). - Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary - Note 1 : Prefix buffer is referenced. It **must** outlive compression. - Its content must remain unmodified during compression. - Note 2 : If the intention is to diff some large src data blob with some prior version of itself, - ensure that the window size is large enough to contain the entire source. - See ZSTD_c_windowLog. - Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. - It's a CPU consuming operation, with non-negligible impact on latency. - If there is a need to use the same prefix multiple times, consider loadDictionary instead. - Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent). - Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. -</p></pre><BR> - <pre><b>typedef enum { ZSTD_reset_session_only = 1, ZSTD_reset_parameters = 2, @@ -672,17 +424,146 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); </p></pre><BR> +<a name="Chapter8"></a><h2>Advanced decompression API</h2><pre></pre> + <pre><b>typedef enum { + + ZSTD_d_windowLogMax=100, </b>/* Select a size limit (in power of 2) beyond which<b> + * the streaming API will refuse to allocate memory buffer + * in order to protect the host from unreasonable memory requirements. + * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. + * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT). + * Special: value 0 means "use default maximum windowLog". */ + + </b>/* note : additional experimental parameters are also available<b> + * within the experimental section of the API. + * At the time of this writing, they include : + * ZSTD_c_format + * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. + * note : never ever use experimentalParam? names directly + */ + ZSTD_d_experimentalParam1=1000 + +} ZSTD_dParameter; +</b></pre><BR> +<pre><b>ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam); +</b><p> All parameters must belong to an interval with lower and upper bounds, + otherwise they will either trigger an error or be automatically clamped. + @return : a structure, ZSTD_bounds, which contains + - an error status field, which must be tested using ZSTD_isError() + - both lower and upper bounds, inclusive + +</p></pre><BR> + +<pre><b>size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value); +</b><p> Set one compression parameter, selected by enum ZSTD_dParameter. + All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds(). + Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). + Setting a parameter is only possible during frame initialization (before starting decompression). + @return : 0, or an error code (which can be tested using ZSTD_isError()). + +</p></pre><BR> + +<pre><b>size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset); +</b><p> Return a DCtx to clean state. + Session and parameters can be reset jointly or separately. + Parameters can only be reset when no active frame is being decompressed. + @return : 0, or an error code, which can be tested with ZSTD_isError() + +</p></pre><BR> + +<a name="Chapter9"></a><h2>Streaming</h2><pre></pre> + +<pre><b>typedef struct ZSTD_inBuffer_s { + const void* src; </b>/**< start of input buffer */<b> + size_t size; </b>/**< size of input buffer */<b> + size_t pos; </b>/**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */<b> +} ZSTD_inBuffer; +</b></pre><BR> +<pre><b>typedef struct ZSTD_outBuffer_s { + void* dst; </b>/**< start of output buffer */<b> + size_t size; </b>/**< size of output buffer */<b> + size_t pos; </b>/**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */<b> +} ZSTD_outBuffer; +</b></pre><BR> +<a name="Chapter10"></a><h2>Streaming compression - HowTo</h2><pre> + A ZSTD_CStream object is required to track streaming operation. + Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. + ZSTD_CStream objects can be reused multiple times on consecutive compression operations. + It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory. + + For parallel execution, use one separate ZSTD_CStream per thread. + + note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing. + + Parameters are sticky : when starting a new compression on the same context, + it will re-use the same sticky parameters as previous compression session. + When in doubt, it's recommended to fully initialize the context before usage. + Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(), + ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to + set more specific parameters, the pledged source size, or load a dictionary. + + Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to + consume input stream. The function will automatically update both `pos` + fields within `input` and `output`. + Note that the function may not consume the entire input, for example, because + the output buffer is already full, in which case `input.pos < input.size`. + The caller must check if input has been entirely consumed. + If not, the caller must make some room to receive more compressed data, + and then present again remaining input data. + note: ZSTD_e_continue is guaranteed to make some forward progress when called, + but doesn't guarantee maximal forward progress. This is especially relevant + when compressing with multiple threads. The call won't block if it can + consume some input, but if it can't it will wait for some, but not all, + output to be flushed. + @return : provides a minimum amount of data remaining to be flushed from internal buffers + or an error code, which can be tested using ZSTD_isError(). + + At any moment, it's possible to flush whatever data might remain stuck within internal buffer, + using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated. + Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0). + In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush. + You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the + operation. + note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will + block until the flush is complete or the output buffer is full. + @return : 0 if internal buffers are entirely flushed, + >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), + or an error code, which can be tested using ZSTD_isError(). + + Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame. + It will perform a flush and write frame epilogue. + The epilogue is required for decoders to consider a frame completed. + flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush. + You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to + start a new frame. + note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will + block until the flush is complete or the output buffer is full. + @return : 0 if frame fully completed and fully flushed, + >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), + or an error code, which can be tested using ZSTD_isError(). + + +<BR></pre> + +<pre><b>typedef ZSTD_CCtx ZSTD_CStream; </b>/**< CCtx and CStream are now effectively same object (>= v1.3.0) */<b> +</b></pre><BR> +<h3>ZSTD_CStream management functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream(void); +size_t ZSTD_freeCStream(ZSTD_CStream* zcs); +</pre></b><BR> +<h3>Streaming compression functions</h3><pre></pre><b><pre>typedef enum { ZSTD_e_continue=0, </b>/* collect more data, encoder decides when to output compressed result, for optimal compression ratio */<b> ZSTD_e_flush=1, </b>/* flush any data provided so far,<b> * it creates (at least) one new block, that can be decoded immediately on reception; - * frame will continue: any future data can still reference previously compressed data, improving compression. */ + * frame will continue: any future data can still reference previously compressed data, improving compression. + * note : multithreaded compression will block to flush as much output as possible. */ ZSTD_e_end=2 </b>/* flush any remaining data _and_ close current frame.<b> * note that frame is only closed after compressed data is fully flushed (return value == 0). * After that point, any additional data starts a new frame. - * note : each frame is independent (does not reference any content from previous frame). */ + * note : each frame is independent (does not reference any content from previous frame). + : note : multithreaded compression will block to flush as much output as possible. */ } ZSTD_EndDirective; -</b></pre><BR> +</pre></b><BR> <pre><b>size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input, @@ -690,8 +571,8 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); </b><p> Behaves about the same as ZSTD_compressStream, with additional control on end directive. - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode) - - outpot->pos must be <= dstCapacity, input->pos must be <= srcSize - - outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. + - output->pos must be <= dstCapacity, input->pos must be <= srcSize + - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller. - When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available, and then immediately returns, just indicating that there is some data remaining to be flushed. @@ -709,41 +590,224 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); </p></pre><BR> -<pre><b>typedef enum { +<pre><b>size_t ZSTD_CStreamInSize(void); </b>/**< recommended size for input buffer */<b> +</b></pre><BR> +<pre><b>size_t ZSTD_CStreamOutSize(void); </b>/**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */<b> +</b></pre><BR> +<a name="Chapter11"></a><h2>This is a legacy streaming API, and can be replaced by ZSTD_CCtx_reset() and</h2><pre> ZSTD_compressStream2(). It is redundent, but is still fully supported. + Advanced parameters and dictionary compression can only be used through the + new API. +<BR></pre> - ZSTD_d_windowLogMax=100, </b>/* Select a size limit (in power of 2) beyond which<b> - * the streaming API will refuse to allocate memory buffer - * in order to protect the host from unreasonable memory requirements. - * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. - * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) */ +<a name="Chapter12"></a><h2>Equivalent to:</h2><pre> + ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + +<BR></pre> - </b>/* note : additional experimental parameters are also available<b> - * within the experimental section of the API. - * At the time of this writing, they include : - * ZSTD_c_format - * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. - * note : never ever use experimentalParam? names directly - */ - ZSTD_d_experimentalParam1=1000 +<a name="Chapter13"></a><h2>Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).</h2><pre> NOTE: The return value is different. ZSTD_compressStream() returns a hint for + the next read size (if non-zero and not an error). ZSTD_compressStream2() + returns the number of bytes left to flush (if non-zero and not an error). + +<BR></pre> -} ZSTD_dParameter; -</b></pre><BR> -<pre><b>ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam); -</b><p> All parameters must belong to an interval with lower and upper bounds, - otherwise they will either trigger an error or be automatically clamped. - @return : a structure, ZSTD_bounds, which contains - - an error status field, which must be tested using ZSTD_isError() - - both lower and upper bounds, inclusive +<a name="Chapter14"></a><h2>Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush).</h2><pre></pre> + +<a name="Chapter15"></a><h2>Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end).</h2><pre></pre> + +<a name="Chapter16"></a><h2>Streaming decompression - HowTo</h2><pre> + A ZSTD_DStream object is required to track streaming operations. + Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. + ZSTD_DStream objects can be re-used multiple times. + + Use ZSTD_initDStream() to start a new decompression operation. + @return : recommended first input size + Alternatively, use advanced API to set specific properties. + + Use ZSTD_decompressStream() repetitively to consume your input. + The function will update both `pos` fields. + If `input.pos < input.size`, some input has not been consumed. + It's up to the caller to present again remaining data. + The function tries to flush all data decoded immediately, respecting output buffer size. + If `output.pos < output.size`, decoder has flushed everything it could. + But if `output.pos == output.size`, there might be some data left within internal buffers., + In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer. + Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX. + @return : 0 when a frame is completely decoded and fully flushed, + or an error code, which can be tested using ZSTD_isError(), + or any other value > 0, which means there is still some decoding or flushing to do to complete current frame : + the return value is a suggested next input size (just a hint for better latency) + that will never request more than the remaining frame size. +<BR></pre> + +<pre><b>typedef ZSTD_DCtx ZSTD_DStream; </b>/**< DCtx and DStream are now effectively same object (>= v1.3.0) */<b> +</b></pre><BR> +<h3>ZSTD_DStream management functions</h3><pre></pre><b><pre>ZSTD_DStream* ZSTD_createDStream(void); +size_t ZSTD_freeDStream(ZSTD_DStream* zds); +</pre></b><BR> +<h3>Streaming decompression functions</h3><pre></pre><b><pre>size_t ZSTD_initDStream(ZSTD_DStream* zds); +size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); +</pre></b><BR> +<pre><b>size_t ZSTD_DStreamInSize(void); </b>/*!< recommended size for input buffer */<b> +</b></pre><BR> +<pre><b>size_t ZSTD_DStreamOutSize(void); </b>/*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */<b> +</b></pre><BR> +<a name="Chapter17"></a><h2>Simple dictionary API</h2><pre></pre> + +<pre><b>size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + int compressionLevel); +</b><p> Compression at an explicit compression level using a Dictionary. + A dictionary can be any arbitrary data segment (also called a prefix), + or a buffer with specified information (see dictBuilder/zdict.h). + Note : This function loads the dictionary, resulting in significant startup delay. + It's intended for a dictionary used only once. + Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. </p></pre><BR> -<pre><b>size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value); -</b><p> Set one compression parameter, selected by enum ZSTD_dParameter. - All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds(). - Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). - Setting a parameter is only possible during frame initialization (before starting decompression). - @return : 0, or an error code (which can be tested using ZSTD_isError()). - +<pre><b>size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize); +</b><p> Decompression using a known Dictionary. + Dictionary must be identical to the one used during compression. + Note : This function loads the dictionary, resulting in significant startup delay. + It's intended for a dictionary used only once. + Note : When `dict == NULL || dictSize < 8` no dictionary is used. +</p></pre><BR> + +<a name="Chapter18"></a><h2>Bulk processing dictionary API</h2><pre></pre> + +<pre><b>ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, + int compressionLevel); +</b><p> When compressing multiple messages / blocks using the same dictionary, it's recommended to load it only once. + ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup cost. + ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. + `dictBuffer` can be released after ZSTD_CDict creation, because its content is copied within CDict. + Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate `dictBuffer` content. + Note : A ZSTD_CDict can be created from an empty dictBuffer, but it is inefficient when used to compress small data. +</p></pre><BR> + +<pre><b>size_t ZSTD_freeCDict(ZSTD_CDict* CDict); +</b><p> Function frees memory allocated by ZSTD_createCDict(). +</p></pre><BR> + +<pre><b>size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict); +</b><p> Compression using a digested Dictionary. + Recommended when same dictionary is used multiple times. + Note : compression level is _decided at dictionary creation time_, + and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) +</p></pre><BR> + +<pre><b>ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); +</b><p> Create a digested dictionary, ready to start decompression operation without startup delay. + dictBuffer can be released after DDict creation, as its content is copied inside DDict. +</p></pre><BR> + +<pre><b>size_t ZSTD_freeDDict(ZSTD_DDict* ddict); +</b><p> Function frees memory allocated with ZSTD_createDDict() +</p></pre><BR> + +<pre><b>size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict); +</b><p> Decompression using a digested Dictionary. + Recommended when same dictionary is used multiple times. +</p></pre><BR> + +<a name="Chapter19"></a><h2>Dictionary helper functions</h2><pre></pre> + +<pre><b>unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); +</b><p> Provides the dictID stored within dictionary. + if @return == 0, the dictionary is not conformant with Zstandard specification. + It can still be loaded, but as a content-only dictionary. +</p></pre><BR> + +<pre><b>unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); +</b><p> Provides the dictID of the dictionary loaded into `ddict`. + If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + Non-conformant dictionaries can still be loaded, but as content-only dictionaries. +</p></pre><BR> + +<pre><b>unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); +</b><p> Provides the dictID required to decompressed the frame stored within `src`. + If @return == 0, the dictID could not be decoded. + This could for one of the following reasons : + - The frame does not require a dictionary to be decoded (most common case). + - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information. + Note : this use case also happens when using a non-conformant dictionary. + - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). + - This is not a Zstandard frame. + When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. +</p></pre><BR> + +<a name="Chapter20"></a><h2>Advanced dictionary and prefix API</h2><pre> + This API allows dictionaries to be used with ZSTD_compress2(), + ZSTD_compressStream2(), and ZSTD_decompress(). Dictionaries are sticky, and + only reset with the context is reset with ZSTD_reset_parameters or + ZSTD_reset_session_and_parameters. Prefixes are single-use. +<BR></pre> + +<pre><b>size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); +</b><p> Create an internal CDict from `dict` buffer. + Decompression will have to use same dictionary. + @result : 0, or an error code (which can be tested with ZSTD_isError()). + Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, + meaning "return to no-dictionary mode". + Note 1 : Dictionary is sticky, it will be used for all future compressed frames. + To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters). + Note 2 : Loading a dictionary involves building tables. + It's also a CPU consuming operation, with non-negligible impact on latency. + Tables are dependent on compression parameters, and for this reason, + compression parameters can no longer be changed after loading a dictionary. + Note 3 :`dict` content will be copied internally. + Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead. + In such a case, dictionary buffer must outlive its users. + Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() + to precisely select how dictionary content must be interpreted. +</p></pre><BR> + +<pre><b>size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); +</b><p> Reference a prepared dictionary, to be used for all next compressed frames. + Note that compression parameters are enforced from within CDict, + and supersede any compression parameter previously set within CCtx. + The parameters ignored are labled as "superseded-by-cdict" in the ZSTD_cParameter enum docs. + The ignored parameters will be used again if the CCtx is returned to no-dictionary mode. + The dictionary will remain valid for future compressed frames using same CCtx. + @result : 0, or an error code (which can be tested with ZSTD_isError()). + Special : Referencing a NULL CDict means "return to no-dictionary mode". + Note 1 : Currently, only one dictionary can be managed. + Referencing a new dictionary effectively "discards" any previous one. + Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. +</p></pre><BR> + +<pre><b>size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, + const void* prefix, size_t prefixSize); +</b><p> Reference a prefix (single-usage dictionary) for next compressed frame. + A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end). + Decompression will need same prefix to properly regenerate data. + Compressing with a prefix is similar in outcome as performing a diff and compressing it, + but performs much faster, especially during decompression (compression speed is tunable with compression level). + @result : 0, or an error code (which can be tested with ZSTD_isError()). + Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary + Note 1 : Prefix buffer is referenced. It **must** outlive compression. + Its content must remain unmodified during compression. + Note 2 : If the intention is to diff some large src data blob with some prior version of itself, + ensure that the window size is large enough to contain the entire source. + See ZSTD_c_windowLog. + Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. + It's a CPU consuming operation, with non-negligible impact on latency. + If there is a need to use the same prefix multiple times, consider loadDictionary instead. + Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent). + Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. </p></pre><BR> <pre><b>size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); @@ -793,15 +857,25 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); </p></pre><BR> -<pre><b>size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset); -</b><p> Return a DCtx to clean state. - Session and parameters can be reset jointly or separately. - Parameters can only be reset when no active frame is being decompressed. - @return : 0, or an error code, which can be tested with ZSTD_isError() - +<pre><b>size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); +size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); +size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); +size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); +size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); +size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); +</b><p> These functions give the _current_ memory usage of selected object. + Note that object memory usage can evolve (increase or decrease) over time. </p></pre><BR> -<a name="Chapter14"></a><h2>experimental API (static linking only)</h2><pre> +<a name="Chapter21"></a><h2>ADVANCED AND EXPERIMENTAL FUNCTIONS</h2><pre> + The definitions in the following section are considered experimental. + They are provided for advanced scenarios. + They should never be used with a dynamic library, as prototypes may change in the future. + Use them only in association with static linking. + +<BR></pre> + +<a name="Chapter22"></a><h2>experimental API (static linking only)</h2><pre> The following symbols and constants are not planned to join "stable API" status in the near future. They can still change in future versions. @@ -890,12 +964,21 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); ZSTD_dictForceCopy = 2, </b>/* Always copy the dictionary. */<b> } ZSTD_dictAttachPref_e; </b></pre><BR> -<a name="Chapter15"></a><h2>Frame size functions</h2><pre></pre> +<pre><b>typedef enum { + ZSTD_lcm_auto = 0, </b>/**< Automatically determine the compression mode based on the compression level.<b> + * Negative compression levels will be uncompressed, and positive compression + * levels will be compressed. */ + ZSTD_lcm_huffman = 1, </b>/**< Always attempt Huffman compression. Uncompressed literals will still be<b> + * emitted if Huffman compression is not profitable. */ + ZSTD_lcm_uncompressed = 2, </b>/**< Always emit uncompressed literals. */<b> +} ZSTD_literalCompressionMode_e; +</b></pre><BR> +<a name="Chapter23"></a><h2>Frame size functions</h2><pre></pre> <pre><b>unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); -</b><p> `src` should point the start of a series of ZSTD encoded and/or skippable frames +</b><p> `src` should point to the start of a series of ZSTD encoded and/or skippable frames `srcSize` must be the _exact_ size of this series - (i.e. there should be a frame boundary exactly at `srcSize` bytes after `src`) + (i.e. there should be a frame boundary at `src + srcSize`) @return : - decompressed size of all data in all successive frames - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN - if an error occurred: ZSTD_CONTENTSIZE_ERROR @@ -915,13 +998,27 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); however it does mean that all frame data must be present and valid. </p></pre><BR> +<a name="Chapter24"></a><h2>ZSTD_decompressBound() :</h2><pre> `src` should point to the start of a series of ZSTD encoded and/or skippable frames + `srcSize` must be the _exact_ size of this series + (i.e. there should be a frame boundary at `src + srcSize`) + @return : - upper-bound for the decompressed size of all data in all successive frames + - if an error occured: ZSTD_CONTENTSIZE_ERROR + + note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame. + note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`. + in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value. + note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by: + upper-bound = # blocks * min(128 KB, Window_Size) + +<BR></pre> + <pre><b>size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); </b><p> srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX. @return : size of the Frame Header, or an error code (if srcSize is too small) </p></pre><BR> -<a name="Chapter16"></a><h2>Memory management</h2><pre></pre> +<a name="Chapter25"></a><h2>Memory management</h2><pre></pre> <pre><b>size_t ZSTD_estimateCCtxSize(int compressionLevel); size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); @@ -933,7 +1030,7 @@ size_t ZSTD_estimateDCtxSize(void); It will also consider src size to be arbitrarily "large", which is worst case. If srcSize is known to always be small, ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation. ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. - ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. + ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. Note : CCtx size estimation is only correct for single-threaded compression. </p></pre><BR> @@ -946,7 +1043,7 @@ size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize); It will also consider src size to be arbitrarily "large", which is worst case. If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation. ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. - ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. + ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. Note : CStream size estimation is only correct for single-threaded compression. ZSTD_DStream memory budget depends on window Size. This information can be passed manually, using ZSTD_estimateDStreamSize, @@ -1001,7 +1098,7 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; </b>/**< t </p></pre><BR> -<a name="Chapter17"></a><h2>Advanced compression functions</h2><pre></pre> +<a name="Chapter26"></a><h2>Advanced compression functions</h2><pre></pre> <pre><b>ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); </b><p> Create a digested dictionary for compression @@ -1011,22 +1108,26 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; </b>/**< t </p></pre><BR> <pre><b>ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); -</b><p> @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. - `estimatedSrcSize` value is optional, select 0 if not known +</b><p> @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. + `estimatedSrcSize` value is optional, select 0 if not known </p></pre><BR> <pre><b>ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); -</b><p> same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. - All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 +</b><p> same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. + All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 </p></pre><BR> <pre><b>size_t ZSTD_checkCParams(ZSTD_compressionParameters params); -</b><p> Ensure param values remain within authorized range +</b><p> Ensure param values remain within authorized range. + @return 0 on success, or an error code (can be checked with ZSTD_isError()) </p></pre><BR> <pre><b>ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); </b><p> optimize params for a given `srcSize` and `dictSize`. - both values are optional, select `0` if unknown. + `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN. + `dictSize` must be `0` when there is no dictionary. + cPar can be invalid : all parameters will be clamped within valid range in the @return struct. + This function never fails (wide contract) </p></pre><BR> <pre><b>size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, @@ -1072,10 +1173,10 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; </b>/**< t size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); </b><p> Quick howto : - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure - - ZSTD_CCtxParam_setParameter() : Push parameters one by one into - an existing ZSTD_CCtx_params structure. - This is similar to - ZSTD_CCtx_setParameter(). + - ZSTD_CCtxParams_setParameter() : Push parameters one by one into + an existing ZSTD_CCtx_params structure. + This is similar to + ZSTD_CCtx_setParameter(). - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to an existing CCtx. These parameters will be applied to @@ -1105,7 +1206,7 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); </p></pre><BR> -<pre><b>size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); +<pre><b>size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); </b><p> Similar to ZSTD_CCtx_setParameter. Set one compression parameter, selected by enum ZSTD_cParameter. Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams(). @@ -1113,7 +1214,7 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); </p></pre><BR> -<pre><b>size_t ZSTD_CCtxParam_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); +<pre><b>size_t ZSTD_CCtxParams_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); </b><p> Similar to ZSTD_CCtx_getParameter. Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. @result : 0, or an error code (which can be tested with ZSTD_isError()). @@ -1142,7 +1243,7 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); </p></pre><BR> -<a name="Chapter18"></a><h2>Advanced decompression functions</h2><pre></pre> +<a name="Chapter27"></a><h2>Advanced decompression functions</h2><pre></pre> <pre><b>unsigned ZSTD_isFrame(const void* buffer, size_t size); </b><p> Tells if the content of `buffer` starts with a valid Frame Identifier. @@ -1158,30 +1259,6 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); it must remain read accessible throughout the lifetime of DDict </p></pre><BR> -<pre><b>unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); -</b><p> Provides the dictID stored within dictionary. - if @return == 0, the dictionary is not conformant with Zstandard specification. - It can still be loaded, but as a content-only dictionary. -</p></pre><BR> - -<pre><b>unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); -</b><p> Provides the dictID of the dictionary loaded into `ddict`. - If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. - Non-conformant dictionaries can still be loaded, but as content-only dictionaries. -</p></pre><BR> - -<pre><b>unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); -</b><p> Provides the dictID required to decompressed the frame stored within `src`. - If @return == 0, the dictID could not be decoded. - This could for one of the following reasons : - - The frame does not require a dictionary to be decoded (most common case). - - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information. - Note : this use case also happens when using a non-conformant dictionary. - - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). - - This is not a Zstandard frame. - When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. -</p></pre><BR> - <pre><b>size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); </b><p> Same as ZSTD_DCtx_loadDictionary(), but references `dict` content instead of copying it into `dctx`. @@ -1228,20 +1305,74 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); </p></pre><BR> -<a name="Chapter19"></a><h2>Advanced streaming functions</h2><pre> Warning : most of these functions are now redundant with the Advanced API. +<a name="Chapter28"></a><h2>Advanced streaming functions</h2><pre> Warning : most of these functions are now redundant with the Advanced API. Once Advanced API reaches "stable" status, redundant functions will be deprecated, and then at some point removed. <BR></pre> -<h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, "0" also disables frame content size field. It may be enabled in the future. */<b> -size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); </b>/**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/<b> +<h3>Advanced Streaming compression functions</h3><pre></pre><b><pre></b>/**! ZSTD_initCStream_srcSize() :<b> + * This function is deprecated, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * + * pledgedSrcSize must be correct. If it is not known at init time, use + * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, + * "0" also disables frame content size field. It may be enabled in the future. + */ +size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); +</b>/**! ZSTD_initCStream_usingDict() :<b> + * This function is deprecated, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * Creates of an internal CDict (incompatible with static CCtx), except if + * dict == NULL or dictSize < 8, in which case no dict is used. + * Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if + * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. + */ +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); +</b>/**! ZSTD_initCStream_advanced() :<b> + * This function is deprecated, and is approximately equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setZstdParams(zcs, params); // Set the zstd params and leave the rest as-is + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * pledgedSrcSize must be correct. If srcSize is not known at init time, use + * value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. + */ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, - ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */<b> -size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); </b>/**< note : cdict will just be referenced, and must outlive compression session */<b> -size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); </b>/**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters. pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. */<b> + ZSTD_parameters params, unsigned long long pledgedSrcSize); +</b>/**! ZSTD_initCStream_usingCDict() :<b> + * This function is deprecated, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * note : cdict will just be referenced, and must outlive compression session + */ +size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); +</b>/**! ZSTD_initCStream_usingCDict_advanced() :<b> + * This function is deprecated, and is approximately equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setZstdFrameParams(zcs, fParams); // Set the zstd frame params and leave the rest as-is + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * same as ZSTD_initCStream_usingCDict(), with control over frame parameters. + * pledgedSrcSize must be correct. If srcSize is not known at init time, use + * value ZSTD_CONTENTSIZE_UNKNOWN. + */ +size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); </pre></b><BR> <pre><b>size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); -</b><p> start a new frame, using same parameters from previous frame. +</b><p> This function is deprecated, and is equivalent to: + ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + + start a new frame, using same parameters from previous frame. This is typically useful to skip dictionary loading stage, since it will re-use it in-place. Note that zcs must be init at least once before using ZSTD_resetCStream(). If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. @@ -1280,14 +1411,14 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); </b>/**< note : ddict is referenced, it must outlive decompression session */<b> size_t ZSTD_resetDStream(ZSTD_DStream* zds); </b>/**< re-use decompression parameters from previous init; saves dictionary loading */<b> </pre></b><BR> -<a name="Chapter20"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre> +<a name="Chapter29"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre> This is an advanced API, giving full control over buffer management, for users which need direct control over memory. But it's also a complex one, with several restrictions, documented below. Prefer normal streaming API for an easier experience. <BR></pre> -<a name="Chapter21"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre> +<a name="Chapter30"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre> A ZSTD_CCtx object is required to track streaming operations. Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. ZSTD_CCtx object can be re-used multiple times within successive compression operations. @@ -1323,7 +1454,7 @@ size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); </b>/* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */<b> size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */<b> </pre></b><BR> -<a name="Chapter22"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre> +<a name="Chapter31"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre> A ZSTD_DCtx object is required to track streaming operations. Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. A ZSTD_DCtx object can be re-used multiple times. @@ -1405,7 +1536,7 @@ typedef struct { unsigned checksumFlag; } ZSTD_frameHeader; </pre></b><BR> -<a name="Chapter23"></a><h2>ZSTD_getFrameHeader() :</h2><pre> decode Frame Header, or requires larger `srcSize`. +<a name="Chapter32"></a><h2>ZSTD_getFrameHeader() :</h2><pre> decode Frame Header, or requires larger `srcSize`. @return : 0, `zfhPtr` is correctly filled, >0, `srcSize` is too small, value is wanted `srcSize` amount, or an error code, which can be tested using ZSTD_isError() @@ -1421,7 +1552,7 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long <pre><b>typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; </b></pre><BR> -<a name="Chapter24"></a><h2>Block level API</h2><pre></pre> +<a name="Chapter33"></a><h2>Block level API</h2><pre></pre> <pre><b></b><p> Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes). User will have to take in charge required information to regenerate data, such as compressed and content sizes. |