diff options
Diffstat (limited to 'tests')
29 files changed, 1977 insertions, 671 deletions
diff --git a/tests/Makefile b/tests/Makefile index 25bd5c84ef90..f11b731835ea 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -33,7 +33,7 @@ endif CFLAGS ?= -O3 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ - -Wstrict-prototypes -Wundef -Wformat-security \ + -Wstrict-prototypes -Wundef \ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ -Wredundant-decls -Wmissing-prototypes CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) @@ -82,7 +82,7 @@ default: fullbench @echo $(ZSTDMT_OBJECTS) all: fullbench fuzzer zstreamtest paramgrill datagen decodecorpus roundTripCrash \ - fullbench-lib + fullbench-lib poolTests all32: fullbench32 fuzzer32 zstreamtest32 @@ -132,18 +132,18 @@ fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP) fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD) fullbench fullbench32 : DEBUGFLAGS = -DNDEBUG # turn off assert() for speed measurements fullbench fullbench32 : $(ZSTD_FILES) -fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c fullbench.c +fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c fullbench.c $(CC) $(FLAGS) $^ -o $@$(EXT) fullbench-lib : CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ fullbench-lib : zstd-staticLib -fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c fullbench.c +fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c fullbench.c $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) $(ZSTDDIR)/libzstd.a # note : broken : requires unavailable symbols fullbench-dll : zstd-dll fullbench-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd -fullbench-dll: $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c fullbench.c +fullbench-dll: $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c $(PRGDIR)/timefn.c fullbench.c # $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) @@ -152,32 +152,32 @@ fuzzer : LDFLAGS += $(MULTITHREAD_LD) fuzzer32: CFLAGS += -m32 fuzzer : $(ZSTDMT_OBJECTS) fuzzer32: $(ZSTD_FILES) -fuzzer fuzzer32 : $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/datagen.c fuzzer.c +fuzzer fuzzer32 : $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c $(CC) $(FLAGS) $^ -o $@$(EXT) fuzzer-dll : zstd-dll fuzzer-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd -fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/datagen.c fuzzer.c +fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) zbufftest : CPPFLAGS += -I$(ZSTDDIR)/deprecated zbufftest : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings -zbufftest : $(ZSTD_OBJECTS) $(ZBUFF_FILES) $(PRGDIR)/util.c $(PRGDIR)/datagen.c zbufftest.c +zbufftest : $(ZSTD_OBJECTS) $(ZBUFF_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c zbufftest.c $(CC) $(FLAGS) $^ -o $@$(EXT) zbufftest32 : CPPFLAGS += -I$(ZSTDDIR)/deprecated zbufftest32 : CFLAGS += -Wno-deprecated-declarations -m32 -zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/util.c $(PRGDIR)/datagen.c zbufftest.c +zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c zbufftest.c $(CC) $(FLAGS) $^ -o $@$(EXT) zbufftest-dll : zstd-dll zbufftest-dll : CPPFLAGS += -I$(ZSTDDIR)/deprecated zbufftest-dll : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings zbufftest-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd -zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/datagen.c zbufftest.c +zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c zbufftest.c $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) -ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c $(PRGDIR)/util.c seqgen.c zstreamtest.c +ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c seqgen.c zstreamtest.c ZSTREAM_PROPER_FILES := $(ZDICT_FILES) $(ZSTREAM_LOCAL_FILES) ZSTREAMFILES := $(ZSTD_FILES) $(ZSTREAM_PROPER_FILES) zstreamtest32 : CFLAGS += -m32 @@ -203,7 +203,7 @@ zstreamtest-dll : $(ZSTREAM_LOCAL_FILES) $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) paramgrill : DEBUGFLAGS = # turn off assert() by default for speed measurements -paramgrill : $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/benchfn.c $(PRGDIR)/benchzstd.c $(PRGDIR)/datagen.c paramgrill.c +paramgrill : $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(PRGDIR)/benchzstd.c $(PRGDIR)/datagen.c paramgrill.c $(CC) $(FLAGS) $^ -lm -o $@$(EXT) datagen : $(PRGDIR)/datagen.c datagencli.c @@ -222,7 +222,7 @@ legacy : CPPFLAGS += -I$(ZSTDDIR)/legacy -DZSTD_LEGACY_SUPPORT=4 legacy : $(ZSTD_FILES) $(wildcard $(ZSTDDIR)/legacy/*.c) legacy.c $(CC) $(FLAGS) $^ -o $@$(EXT) -decodecorpus : $(filter-out zstdc_zstd_compress.o, $(ZSTD_OBJECTS)) $(ZDICT_FILES) $(PRGDIR)/util.c decodecorpus.c +decodecorpus : $(filter-out zstdc_zstd_compress.o, $(ZSTD_OBJECTS)) $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c decodecorpus.c $(CC) $(FLAGS) $^ -o $@$(EXT) -lm symbols : symbols.c zstd-dll @@ -233,7 +233,7 @@ else $(CC) $(FLAGS) $< -o $@$(EXT) -Wl,-rpath=$(ZSTDDIR) $(ZSTDDIR)/libzstd.so # broken on Mac endif -poolTests : $(PRGDIR)/util.c poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/error_private.c +poolTests : $(PRGDIR)/util.c $(PRGDIR)/timefn.c poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/error_private.c $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT) .PHONY: versionsTest @@ -353,8 +353,12 @@ test-gzstd: gzstd $(RM) *.gz *.zst README2.md gz_zstd zstd_gz hello.txt test-zstdgrep: gzstd - @echo a | $(PRGDIR)/zstd | $(PRGDIR)/zstdgrep a - @echo a | $(PRGDIR)/zstd | $(PRGDIR)/zstdgrep b && return 1 || return 0 + -[ -f /tmp/zstdcat ] || ln -s $(PWD)/$(PRGDIR)/zstd /tmp/zstdcat + echo a | $(PRGDIR)/zstd | env ZCAT=/tmp/zstdcat $(PRGDIR)/zstdgrep a + echo a | $(PRGDIR)/zstd | env ZCAT=/tmp/zstdcat $(PRGDIR)/zstdgrep b && return 1 || return 0 + -echo 'hello world' > test.txt && $(PRGDIR)/zstd test.txt + env ZCAT=/tmp/zstdcat $(PRGDIR)/zstdgrep hello test.txt.zst + env ZCAT=/tmp/zstdcat $(PRGDIR)/zstdgrep weird test.txt.zst && return 1 || return 0 test-fullbench: fullbench datagen $(QEMU_SYS) ./fullbench -i1 diff --git a/tests/README.md b/tests/README.md index 7c6fb0db845c..f34501197478 100644 --- a/tests/README.md +++ b/tests/README.md @@ -72,7 +72,7 @@ Command line tool to generate test .zst files. This tool will generate .zst files with checksums, as well as optionally output the corresponding correct uncompressed data for -extra verfication. +extra verification. Example: ``` @@ -123,7 +123,7 @@ Full list of arguments Higher values will make optimizer run longer, more chances to find better solution. memLog : Limits the log of the size of each memotable (1 per strategy). Will use hash tables when state space is larger than max size. Setting memLog = 0 turns off memoization - --display= : specifiy which parameters are included in the output + --display= : specify which parameters are included in the output can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters (Default: display all params available) -P# : generated sample compressibility (when no file is provided) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 17c2c18991bc..9910d3c551a9 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -16,6 +16,7 @@ #include <string.h> #include "util.h" +#include "timefn.h" /* UTIL_clockSpanMicro, SEC_TO_MICRO, UTIL_TIME_INITIALIZER */ #include "zstd.h" #include "zstd_internal.h" #include "mem.h" @@ -513,7 +514,7 @@ static size_t writeLiteralsBlockCompressed(U32* seed, frame_t* frame, size_t con if ((RAND(seed) & 3) || !frame->stats.hufInit) { do { if (RAND(seed) & 3) { - /* add 10 to ensure some compressability */ + /* add 10 to ensure some compressibility */ double const weight = ((RAND(seed) % 90) + 10) / 100.0; DISPLAYLEVEL(5, " distribution weight: %d%%\n", @@ -938,7 +939,9 @@ static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr, FSE_CState_t stateOffsetBits; FSE_CState_t stateLitLength; - CHECK_E(BIT_initCStream(&blockStream, op, oend-op), dstSize_tooSmall); /* not enough space remaining */ + RETURN_ERROR_IF( + ERR_isError(BIT_initCStream(&blockStream, op, oend-op)), + dstSize_tooSmall, "not enough space remaining"); /* first symbols */ FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]); diff --git a/tests/fullbench.c b/tests/fullbench.c index 8644a2e3a136..b06e2edc53a6 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -17,6 +17,7 @@ #include <stdio.h> /* fprintf, fopen, ftello64 */ #include <assert.h> /* assert */ +#include "timefn.h" /* UTIL_clockSpanNano, UTIL_getTime */ #include "mem.h" /* U32 */ #ifndef ZSTD_DLL_IMPORT #include "zstd_internal.h" /* ZSTD_decodeSeqHeaders, ZSTD_blockHeaderSize, blockType_e, KB, MB */ @@ -67,12 +68,6 @@ static const size_t g_sampleSize = 10000000; static unsigned g_nbIterations = NBLOOPS; static double g_compressibility = COMPRESSIBILITY_DEFAULT; -static void BMK_SetNbIterations(unsigned nbLoops) -{ - g_nbIterations = nbLoops; - DISPLAY("- %i iterations -\n", g_nbIterations); -} - /*_******************************************************* * Private functions @@ -316,9 +311,9 @@ static size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize, /*_******************************************************* * Bench functions *********************************************************/ -static size_t benchMem(unsigned benchNb, - const void* src, size_t srcSize, - int cLevel, ZSTD_compressionParameters cparams) +static int benchMem(unsigned benchNb, + const void* src, size_t srcSize, + int cLevel, ZSTD_compressionParameters cparams) { size_t dstBuffSize = ZSTD_compressBound(srcSize); BYTE* dstBuff; @@ -395,22 +390,22 @@ static size_t benchMem(unsigned benchNb, cparams->minMatch, cparams->targetLength, cparams->strategy); */ ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_compressionLevel, cLevel); - ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_windowLog, cparams.windowLog); - ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_hashLog, cparams.hashLog); - ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_chainLog, cparams.chainLog); - ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_searchLog, cparams.searchLog); - ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_minMatch, cparams.minMatch); - ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_targetLength, cparams.targetLength); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_windowLog, (int)cparams.windowLog); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_hashLog, (int)cparams.hashLog); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_chainLog, (int)cparams.chainLog); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_searchLog, (int)cparams.searchLog); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_minMatch, (int)cparams.minMatch); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_targetLength, (int)cparams.targetLength); ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_strategy, cparams.strategy); ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_compressionLevel, cLevel); - ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_windowLog, cparams.windowLog); - ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_hashLog, cparams.hashLog); - ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_chainLog, cparams.chainLog); - ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_searchLog, cparams.searchLog); - ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_minMatch, cparams.minMatch); - ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_targetLength, cparams.targetLength); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_windowLog, (int)cparams.windowLog); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_hashLog, (int)cparams.hashLog); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_chainLog, (int)cparams.chainLog); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_searchLog, (int)cparams.searchLog); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_minMatch, (int)cparams.minMatch); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_targetLength, (int)cparams.targetLength); ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_strategy, cparams.strategy); /* Preparation */ @@ -469,8 +464,9 @@ static size_t benchMem(unsigned benchNb, iend = ip + ZSTD_blockHeaderSize + cBlockSize; /* End of first block */ ip += ZSTD_blockHeaderSize; /* skip block header */ ZSTD_decompressBegin(g_zdc); - ip += ZSTD_decodeLiteralsBlock(g_zdc, ip, iend-ip); /* skip literal segment */ - g_cSize = iend-ip; + assert(iend > ip); + ip += ZSTD_decodeLiteralsBlock(g_zdc, ip, (size_t)(iend-ip)); /* skip literal segment */ + g_cSize = (size_t)(iend-ip); memcpy(buff2, ip, g_cSize); /* copy rest of block (it starts by SeqHeader) */ srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */ break; @@ -501,7 +497,7 @@ static size_t benchMem(unsigned benchNb, BMK_benchParams_t bp; BMK_runTime_t bestResult; bestResult.sumOfReturn = 0; - bestResult.nanoSecPerRun = (unsigned long long)(-1LL); + bestResult.nanoSecPerRun = (double)TIMELOOP_NANOSEC * 2000000000; /* hopefully large enough : must be larger than any potential measurement */ assert(tfs != NULL); bp.benchFn = benchFunction; @@ -654,7 +650,9 @@ static unsigned readU32FromChar(const char** stringPtr) while ((**stringPtr >='0') && (**stringPtr <='9')) { unsigned const max = (((unsigned)(-1)) / 10) - 1; if (result > max) ERROR_OUT(errorMsg); - result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + result *= 10; + result += (unsigned)(**stringPtr - '0'); + (*stringPtr)++ ; } if ((**stringPtr=='K') || (**stringPtr=='M')) { unsigned const maxK = ((unsigned)(-1)) >> 10; @@ -671,7 +669,7 @@ static unsigned readU32FromChar(const char** stringPtr) return result; } -static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) +static int longCommandWArg(const char** stringPtr, const char* longCommand) { size_t const comSize = strlen(longCommand); int const result = !strncmp(*stringPtr, longCommand, comSize); @@ -772,7 +770,7 @@ int main(int argc, const char** argv) /* Modify Nb Iterations */ case 'i': argument++; - BMK_SetNbIterations((int)readU32FromChar(&argument)); + g_nbIterations = readU32FromChar(&argument); break; /* Select compressibility of synthetic sample */ @@ -782,7 +780,7 @@ int main(int argc, const char** argv) break; case 'l': argument++; - cLevel = readU32FromChar(&argument); + cLevel = (int)readU32FromChar(&argument); cparams = ZSTD_getCParams(cLevel, 0, 0); break; diff --git a/tests/fuzz/Makefile b/tests/fuzz/Makefile index 4130f18e330c..31b151b857b3 100644 --- a/tests/fuzz/Makefile +++ b/tests/fuzz/Makefile @@ -27,16 +27,16 @@ PRGDIR = ../../programs FUZZ_CPPFLAGS := -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) \ - $(CPPFLAGS) + -DZSTD_MULTITHREAD $(CPPFLAGS) FUZZ_EXTRA_FLAGS := -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ - -Wstrict-prototypes -Wundef -Wformat-security \ + -Wstrict-prototypes -Wundef \ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ -Wredundant-decls \ -g -fno-omit-frame-pointer FUZZ_CFLAGS := $(FUZZ_EXTRA_FLAGS) $(CFLAGS) FUZZ_CXXFLAGS := $(FUZZ_EXTRA_FLAGS) -std=c++11 $(CXXFLAGS) -FUZZ_LDFLAGS := $(LDFLAGS) +FUZZ_LDFLAGS := -pthread $(LDFLAGS) FUZZ_ARFLAGS := $(ARFLAGS) FUZZ_TARGET_FLAGS = $(FUZZ_CPPFLAGS) $(FUZZ_CXXFLAGS) $(FUZZ_LDFLAGS) @@ -46,11 +46,13 @@ FUZZ_SRC := $(PRGDIR)/util.c zstd_helpers.c ZSTDCOMMON_SRC := $(ZSTDDIR)/common/*.c ZSTDCOMP_SRC := $(ZSTDDIR)/compress/*.c ZSTDDECOMP_SRC := $(ZSTDDIR)/decompress/*.c +ZSTDDICT_SRC := $(ZSTDDIR)/dictBuilder/*.c FUZZ_SRC := \ $(FUZZ_SRC) \ $(ZSTDDECOMP_SRC) \ $(ZSTDCOMMON_SRC) \ - $(ZSTDCOMP_SRC) + $(ZSTDCOMP_SRC) \ + $(ZSTDDICT_SRC) FUZZ_OBJ := $(patsubst %.c,%.o, $(wildcard $(FUZZ_SRC))) @@ -65,7 +67,9 @@ FUZZ_TARGETS := \ block_round_trip \ simple_decompress \ stream_decompress \ - block_decompress + block_decompress \ + dictionary_round_trip \ + dictionary_decompress all: $(FUZZ_TARGETS) @@ -90,11 +94,17 @@ stream_decompress: $(FUZZ_HEADERS) $(FUZZ_OBJ) stream_decompress.o block_decompress: $(FUZZ_HEADERS) $(FUZZ_OBJ) block_decompress.o $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) block_decompress.o $(LIB_FUZZING_ENGINE) -o $@ +dictionary_round_trip: $(FUZZ_HEADERS) $(FUZZ_OBJ) dictionary_round_trip.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) dictionary_round_trip.o $(LIB_FUZZING_ENGINE) -o $@ + +dictionary_decompress: $(FUZZ_HEADERS) $(FUZZ_OBJ) dictionary_decompress.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) dictionary_decompress.o $(LIB_FUZZING_ENGINE) -o $@ + libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h $(PRGDIR)/util.c regression_driver.o $(AR) $(FUZZ_ARFLAGS) $@ regression_driver.o # Install libfuzzer (not usable for MSAN testing) -# Provided for convienence. To use this library run make libFuzzer and +# Provided for convenience. To use this library run make libFuzzer and # set LDFLAGS=-L. .PHONY: libFuzzer libFuzzer: diff --git a/tests/fuzz/README.md b/tests/fuzz/README.md index f184be646ef6..9e0bb259a9e7 100644 --- a/tests/fuzz/README.md +++ b/tests/fuzz/README.md @@ -37,8 +37,8 @@ The specific fuzzing engine is selected with `LIB_FUZZING_ENGINE` or `--lib-fuzzing-engine`, the default is `libregression.a`. It has flags that can easily set up sanitizers `--enable-{a,ub,m}san`, and coverage instrumentation `--enable-coverage`. -It sets sane defaults which can be overriden with flags `--debug`, -`--enable-ubsan-pointer-overlow`, etc. +It sets sane defaults which can be overridden with flags `--debug`, +`--enable-ubsan-pointer-overflow`, etc. Run `./fuzz.py build -h` for help. ### Running Fuzzers diff --git a/tests/fuzz/dictionary_decompress.c b/tests/fuzz/dictionary_decompress.c new file mode 100644 index 000000000000..7d3a7678adc6 --- /dev/null +++ b/tests/fuzz/dictionary_decompress.c @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + */ + +/** + * This fuzz target attempts to decompress the fuzzed data with the dictionary + * decompression function to ensure the decompressor never crashes. It does not + * fuzz the dictionary. + */ + +#include <stddef.h> +#include <stdlib.h> +#include <stdio.h> +#include "fuzz_helpers.h" +#include "zstd_helpers.h" + +static ZSTD_DCtx *dctx = NULL; +static void* rBuf = NULL; +static size_t bufSize = 0; + +int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) +{ + FUZZ_dict_t dict; + size_t neededBufSize; + + uint32_t seed = FUZZ_seed(&src, &size); + neededBufSize = MAX(20 * size, (size_t)256 << 10); + + /* Allocate all buffers and contexts if not already allocated */ + if (neededBufSize > bufSize) { + free(rBuf); + rBuf = malloc(neededBufSize); + bufSize = neededBufSize; + FUZZ_ASSERT(rBuf); + } + if (!dctx) { + dctx = ZSTD_createDCtx(); + FUZZ_ASSERT(dctx); + } + dict = FUZZ_train(src, size, &seed); + if (FUZZ_rand32(&seed, 0, 1) == 0) { + ZSTD_decompress_usingDict(dctx, + rBuf, neededBufSize, + src, size, + dict.buff, dict.size); + } else { + FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary_advanced( + dctx, dict.buff, dict.size, + (ZSTD_dictLoadMethod_e)FUZZ_rand32(&seed, 0, 1), + (ZSTD_dictContentType_e)FUZZ_rand32(&seed, 0, 2))); + ZSTD_decompressDCtx(dctx, rBuf, neededBufSize, src, size); + } + + free(dict.buff); +#ifndef STATEFUL_FUZZING + ZSTD_freeDCtx(dctx); dctx = NULL; +#endif + return 0; +} diff --git a/tests/fuzz/dictionary_round_trip.c b/tests/fuzz/dictionary_round_trip.c new file mode 100644 index 000000000000..e28c65c98f06 --- /dev/null +++ b/tests/fuzz/dictionary_round_trip.c @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + */ + +/** + * This fuzz target performs a zstd round-trip test (compress & decompress) with + * a dictionary, compares the result with the original, and calls abort() on + * corruption. + */ + +#include <stddef.h> +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include "fuzz_helpers.h" +#include "zstd_helpers.h" + +static const int kMaxClevel = 19; + +static ZSTD_CCtx *cctx = NULL; +static ZSTD_DCtx *dctx = NULL; +static uint32_t seed; + +static size_t roundTripTest(void *result, size_t resultCapacity, + void *compressed, size_t compressedCapacity, + const void *src, size_t srcSize) +{ + ZSTD_dictContentType_e dictContentType = ZSTD_dct_auto; + FUZZ_dict_t dict = FUZZ_train(src, srcSize, &seed); + size_t cSize; + if ((FUZZ_rand(&seed) & 15) == 0) { + int const cLevel = FUZZ_rand(&seed) % kMaxClevel; + + cSize = ZSTD_compress_usingDict(cctx, + compressed, compressedCapacity, + src, srcSize, + dict.buff, dict.size, + cLevel); + } else { + dictContentType = FUZZ_rand32(&seed, 0, 2); + FUZZ_setRandomParameters(cctx, srcSize, &seed); + /* Disable checksum so we can use sizes smaller than compress bound. */ + FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 0)); + FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary_advanced( + cctx, dict.buff, dict.size, + (ZSTD_dictLoadMethod_e)FUZZ_rand32(&seed, 0, 1), + dictContentType)); + cSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize); + } + FUZZ_ZASSERT(cSize); + FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary_advanced( + dctx, dict.buff, dict.size, + (ZSTD_dictLoadMethod_e)FUZZ_rand32(&seed, 0, 1), + dictContentType)); + { + size_t const ret = ZSTD_decompressDCtx( + dctx, result, resultCapacity, compressed, cSize); + free(dict.buff); + return ret; + } +} + +int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) +{ + size_t const rBufSize = size; + void* rBuf = malloc(rBufSize); + size_t cBufSize = ZSTD_compressBound(size); + void* cBuf; + + seed = FUZZ_seed(&src, &size); + /* Half of the time fuzz with a 1 byte smaller output size. + * This will still succeed because we force the checksum to be disabled, + * giving us 4 bytes of overhead. + */ + cBufSize -= FUZZ_rand32(&seed, 0, 1); + cBuf = malloc(cBufSize); + + if (!cctx) { + cctx = ZSTD_createCCtx(); + FUZZ_ASSERT(cctx); + } + if (!dctx) { + dctx = ZSTD_createDCtx(); + FUZZ_ASSERT(dctx); + } + + { + size_t const result = + roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size); + FUZZ_ZASSERT(result); + FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size"); + FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!"); + } + free(rBuf); + free(cBuf); +#ifndef STATEFUL_FUZZING + ZSTD_freeCCtx(cctx); cctx = NULL; + ZSTD_freeDCtx(dctx); dctx = NULL; +#endif + return 0; +} diff --git a/tests/fuzz/fuzz.py b/tests/fuzz/fuzz.py index 8ce293a3a695..cd2a5b4d442e 100755 --- a/tests/fuzz/fuzz.py +++ b/tests/fuzz/fuzz.py @@ -34,6 +34,8 @@ TARGETS = [ 'simple_decompress', 'stream_decompress', 'block_decompress', + 'dictionary_round_trip', + 'dictionary_decompress', ] ALL_TARGETS = TARGETS + ['all'] FUZZ_RNG_SEED_SIZE = 4 @@ -192,11 +194,21 @@ def build_parser(args): default=LIB_FUZZING_ENGINE, help=('The fuzzing engine to use e.g. /path/to/libFuzzer.a ' "(default: $LIB_FUZZING_ENGINE='{})".format(LIB_FUZZING_ENGINE))) - parser.add_argument( + + fuzz_group = parser.add_mutually_exclusive_group() + fuzz_group.add_argument( '--enable-coverage', dest='coverage', action='store_true', help='Enable coverage instrumentation (-fsanitize-coverage)') + fuzz_group.add_argument( + '--enable-fuzzer', + dest='fuzzer', + action='store_true', + help=('Enable clang fuzzer (-fsanitize=fuzzer). When enabled ' + 'LIB_FUZZING_ENGINE is ignored') + ) + parser.add_argument( '--enable-asan', dest='asan', action='store_true', help='Enable UBSAN') parser.add_argument( @@ -327,13 +339,13 @@ def build_parser(args): args = parse_env_flags(args, ' '.join( [args.cppflags, args.cflags, args.cxxflags, args.ldflags])) - # Check option sanitiy + # Check option sanity if args.msan and (args.asan or args.ubsan): raise RuntimeError('MSAN may not be used with any other sanitizers') if args.msan_track_origins and not args.msan: raise RuntimeError('--enable-msan-track-origins requires MSAN') if args.ubsan_pointer_overflow and not args.ubsan: - raise RuntimeError('--enable-ubsan-pointer-overlow requires UBSAN') + raise RuntimeError('--enable-ubsan-pointer-overflow requires UBSAN') if args.sanitize_recover and not args.sanitize: raise RuntimeError('--enable-sanitize-recover but no sanitizers used') @@ -364,13 +376,17 @@ def build(args): '-DFUZZ_RNG_SEED_SIZE={}'.format(args.fuzz_rng_seed_size), ] - mflags += ['LIB_FUZZING_ENGINE={}'.format(args.lib_fuzzing_engine)] - # Set flags for options + assert not (args.fuzzer and args.coverage) if args.coverage: common_flags += [ '-fsanitize-coverage=trace-pc-guard,indirect-calls,trace-cmp' ] + if args.fuzzer: + common_flags += ['-fsanitize=fuzzer'] + args.lib_fuzzing_engine = '' + + mflags += ['LIB_FUZZING_ENGINE={}'.format(args.lib_fuzzing_engine)] if args.sanitize_recover: recover_flags = ['-fsanitize-recover=all'] @@ -607,7 +623,7 @@ def regression(args): def gen_parser(args): description = """ - Generate a seed corpus appropiate for TARGET with data generated with + Generate a seed corpus appropriate for TARGET with data generated with decodecorpus. The fuzz inputs are prepended with a seed before the zstd data, so the output of decodecorpus shouldn't be used directly. diff --git a/tests/fuzz/fuzz_helpers.h b/tests/fuzz/fuzz_helpers.h index 468c39fb42d4..0cf79d0d7ce9 100644 --- a/tests/fuzz/fuzz_helpers.h +++ b/tests/fuzz/fuzz_helpers.h @@ -55,7 +55,7 @@ extern "C" { #endif /** - * Determininistically constructs a seed based on the fuzz input. + * Deterministically constructs a seed based on the fuzz input. * Consumes up to the first FUZZ_RNG_SEED_SIZE bytes of the input. */ FUZZ_STATIC uint32_t FUZZ_seed(uint8_t const **src, size_t* size) { diff --git a/tests/fuzz/regression_driver.c b/tests/fuzz/regression_driver.c index 1553d436ce03..658c685f4f89 100644 --- a/tests/fuzz/regression_driver.c +++ b/tests/fuzz/regression_driver.c @@ -40,8 +40,13 @@ int main(int argc, char const **argv) { size_t readSize; FILE *file; - /* Check that it is a regular file, and that the fileSize is valid */ - FUZZ_ASSERT_MSG(UTIL_isRegularFile(fileName), fileName); + /* Check that it is a regular file, and that the fileSize is valid. + * If it is not a regular file, then it may have been deleted since we + * constructed the list, so just skip it. + */ + if (!UTIL_isRegularFile(fileName)) { + continue; + } FUZZ_ASSERT_MSG(fileSize <= kMaxFileSize, fileName); /* Ensure we have a large enough buffer allocated */ if (fileSize > bufferSize) { diff --git a/tests/fuzz/simple_round_trip.c b/tests/fuzz/simple_round_trip.c index 83608b6e79ea..7e3b6609822f 100644 --- a/tests/fuzz/simple_round_trip.c +++ b/tests/fuzz/simple_round_trip.c @@ -25,9 +25,6 @@ static const int kMaxClevel = 19; static ZSTD_CCtx *cctx = NULL; static ZSTD_DCtx *dctx = NULL; -static void* cBuf = NULL; -static void* rBuf = NULL; -static size_t bufSize = 0; static uint32_t seed; static size_t roundTripTest(void *result, size_t resultCapacity, @@ -36,16 +33,8 @@ static size_t roundTripTest(void *result, size_t resultCapacity, { size_t cSize; if (FUZZ_rand(&seed) & 1) { - ZSTD_inBuffer in = {src, srcSize, 0}; - ZSTD_outBuffer out = {compressed, compressedCapacity, 0}; - size_t err; - - ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); FUZZ_setRandomParameters(cctx, srcSize, &seed); - err = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end); - FUZZ_ZASSERT(err); - FUZZ_ASSERT(err == 0); - cSize = out.pos; + cSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize); } else { int const cLevel = FUZZ_rand(&seed) % kMaxClevel; cSize = ZSTD_compressCCtx( @@ -57,20 +46,21 @@ static size_t roundTripTest(void *result, size_t resultCapacity, int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { - size_t neededBufSize; + size_t const rBufSize = size; + void* rBuf = malloc(rBufSize); + size_t cBufSize = ZSTD_compressBound(size); + void* cBuf; seed = FUZZ_seed(&src, &size); - neededBufSize = ZSTD_compressBound(size); + /* Half of the time fuzz with a 1 byte smaller output size. + * This will still succeed because we don't use a dictionary, so the dictID + * field is empty, giving us 4 bytes of overhead. + */ + cBufSize -= FUZZ_rand32(&seed, 0, 1); + cBuf = malloc(cBufSize); + + FUZZ_ASSERT(cBuf && rBuf); - /* Allocate all buffers and contexts if not already allocated */ - if (neededBufSize > bufSize) { - free(cBuf); - free(rBuf); - cBuf = malloc(neededBufSize); - rBuf = malloc(neededBufSize); - bufSize = neededBufSize; - FUZZ_ASSERT(cBuf && rBuf); - } if (!cctx) { cctx = ZSTD_createCCtx(); FUZZ_ASSERT(cctx); @@ -82,11 +72,13 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { size_t const result = - roundTripTest(rBuf, neededBufSize, cBuf, neededBufSize, src, size); + roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size); FUZZ_ZASSERT(result); FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size"); FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!"); } + free(rBuf); + free(cBuf); #ifndef STATEFUL_FUZZING ZSTD_freeCCtx(cctx); cctx = NULL; ZSTD_freeDCtx(dctx); dctx = NULL; diff --git a/tests/fuzz/stream_decompress.c b/tests/fuzz/stream_decompress.c index 7ad571221dff..68e120d7ef6d 100644 --- a/tests/fuzz/stream_decompress.c +++ b/tests/fuzz/stream_decompress.c @@ -62,9 +62,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) if (!dstream) { dstream = ZSTD_createDStream(); FUZZ_ASSERT(dstream); - FUZZ_ASSERT(!ZSTD_isError(ZSTD_initDStream(dstream))); } else { - FUZZ_ASSERT(!ZSTD_isError(ZSTD_resetDStream(dstream))); + FUZZ_ZASSERT(ZSTD_DCtx_reset(dstream, ZSTD_reset_session_only)); } while (size > 0) { @@ -73,7 +72,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) ZSTD_outBuffer out = makeOutBuffer(); size_t const rc = ZSTD_decompressStream(dstream, &out, &in); if (ZSTD_isError(rc)) goto error; - if (rc == 0) FUZZ_ASSERT(!ZSTD_isError(ZSTD_resetDStream(dstream))); } } diff --git a/tests/fuzz/stream_round_trip.c b/tests/fuzz/stream_round_trip.c index d903bcb29b11..d13c2dbe7e00 100644 --- a/tests/fuzz/stream_round_trip.c +++ b/tests/fuzz/stream_round_trip.c @@ -63,7 +63,7 @@ static size_t compress(uint8_t *dst, size_t capacity, ZSTD_inBuffer in = makeInBuffer(&src, &srcSize); /* Mode controls the action. If mode == -1 we pick a new mode */ int mode = -1; - while (in.pos < in.size) { + while (in.pos < in.size || mode != -1) { ZSTD_outBuffer out = makeOutBuffer(dst, capacity); /* Previous action finished, pick a new mode. */ if (mode == -1) mode = FUZZ_rand(&seed) % 10; diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c index 10163e1512b0..9dff2895a9c0 100644 --- a/tests/fuzz/zstd_helpers.c +++ b/tests/fuzz/zstd_helpers.c @@ -8,10 +8,14 @@ */ #define ZSTD_STATIC_LINKING_ONLY +#define ZDICT_STATIC_LINKING_ONLY + +#include <string.h> #include "zstd_helpers.h" #include "fuzz_helpers.h" #include "zstd.h" +#include "zdict.h" static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, int value) { @@ -71,8 +75,7 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state) setRand(cctx, ZSTD_c_contentSizeFlag, 0, 1, state); setRand(cctx, ZSTD_c_checksumFlag, 0, 1, state); setRand(cctx, ZSTD_c_dictIDFlag, 0, 1, state); - setRand(cctx, ZSTD_c_forceAttachDict, 0, 2, state); - /* Select long distance matchig parameters */ + /* Select long distance matching parameters */ setRand(cctx, ZSTD_c_enableLongDistanceMatching, 0, 1, state); setRand(cctx, ZSTD_c_ldmHashLog, ZSTD_HASHLOG_MIN, 16, state); setRand(cctx, ZSTD_c_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN, @@ -81,4 +84,54 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state) state); setRand(cctx, ZSTD_c_ldmHashRateLog, ZSTD_LDM_HASHRATELOG_MIN, ZSTD_LDM_HASHRATELOG_MAX, state); + /* Set misc parameters */ + setRand(cctx, ZSTD_c_nbWorkers, 0, 2, state); + setRand(cctx, ZSTD_c_rsyncable, 0, 1, state); + setRand(cctx, ZSTD_c_forceMaxWindow, 0, 1, state); + setRand(cctx, ZSTD_c_literalCompressionMode, 0, 2, state); + setRand(cctx, ZSTD_c_forceAttachDict, 0, 2, state); +} + +FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, uint32_t *state) +{ + size_t const dictSize = MAX(srcSize / 8, 1024); + size_t const totalSampleSize = dictSize * 11; + FUZZ_dict_t dict = { malloc(dictSize), dictSize }; + char* const samples = (char*)malloc(totalSampleSize); + unsigned nbSamples = 100; + size_t* const samplesSizes = (size_t*)malloc(sizeof(size_t) * nbSamples); + size_t pos = 0; + size_t sample = 0; + ZDICT_fastCover_params_t params; + FUZZ_ASSERT(dict.buff && samples && samplesSizes); + + for (sample = 0; sample < nbSamples; ++sample) { + size_t const remaining = totalSampleSize - pos; + size_t const offset = FUZZ_rand32(state, 0, MAX(srcSize, 1) - 1); + size_t const limit = MIN(srcSize - offset, remaining); + size_t const toCopy = MIN(limit, remaining / (nbSamples - sample)); + memcpy(samples + pos, src + offset, toCopy); + pos += toCopy; + samplesSizes[sample] = toCopy; + + } + memset(samples + pos, 0, totalSampleSize - pos); + + memset(¶ms, 0, sizeof(params)); + params.accel = 5; + params.k = 40; + params.d = 8; + params.f = 14; + params.zParams.compressionLevel = 1; + dict.size = ZDICT_trainFromBuffer_fastCover(dict.buff, dictSize, + samples, samplesSizes, nbSamples, params); + if (ZSTD_isError(dict.size)) { + free(dict.buff); + memset(&dict, 0, sizeof(dict)); + } + + free(samplesSizes); + free(samples); + + return dict; } diff --git a/tests/fuzz/zstd_helpers.h b/tests/fuzz/zstd_helpers.h index 3856bebecf7e..457e6e995f0d 100644 --- a/tests/fuzz/zstd_helpers.h +++ b/tests/fuzz/zstd_helpers.h @@ -14,6 +14,8 @@ #ifndef ZSTD_HELPERS_H #define ZSTD_HELPERS_H +#define ZSTD_STATIC_LINKING_ONLY + #include "zstd.h" #include <stdint.h> @@ -27,6 +29,17 @@ ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state); ZSTD_frameParameters FUZZ_randomFParams(uint32_t *state); ZSTD_parameters FUZZ_randomParams(size_t srcSize, uint32_t *state); +typedef struct { + void* buff; + size_t size; +} FUZZ_dict_t; + +/* Quickly train a dictionary from a source for fuzzing. + * NOTE: Don't use this to train production dictionaries, it is only optimized + * for speed, and doesn't care about dictionary quality. + */ +FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, uint32_t *state); + #ifdef __cplusplus } diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 946844e1614a..1a31c78e2922 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -38,6 +38,7 @@ #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ #include "xxhash.h" /* XXH64 */ #include "util.h" +#include "timefn.h" /* SEC_TO_MICRO, UTIL_time_t, UTIL_TIME_INITIALIZER, UTIL_clockSpanMicro, UTIL_getTime */ /*-************************************ @@ -124,12 +125,14 @@ static U32 FUZ_highbit32(U32 v32) #define CHECK(fn) { CHECK_V(err, fn); } #define CHECKPLUS(var, fn, more) { CHECK_V(var, fn); more; } -#define CHECK_EQ(lhs, rhs) { \ - if ((lhs) != (rhs)) { \ - DISPLAY("Error L%u => %s != %s ", __LINE__, #lhs, #rhs); \ +#define CHECK_OP(op, lhs, rhs) { \ + if (!((lhs) op (rhs))) { \ + DISPLAY("Error L%u => FAILED %s %s %s ", __LINE__, #lhs, #op, #rhs); \ goto _output_error; \ } \ } +#define CHECK_EQ(lhs, rhs) CHECK_OP(==, lhs, rhs) +#define CHECK_LT(lhs, rhs) CHECK_OP(<, lhs, rhs) /*============================================= @@ -374,6 +377,20 @@ static int basicUnitTests(U32 seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : tight ZSTD_decompressBound test : ", testNb++); + { + unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize); + if (bound != CNBuffSize) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with invalid srcSize : ", testNb++); + { + unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize - 1); + if (bound != ZSTD_CONTENTSIZE_ERROR) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize); { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); if (r != CNBuffSize) goto _output_error; } @@ -429,6 +446,27 @@ static int basicUnitTests(U32 seed, double compressibility) if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with content size missing : ", testNb++); + { /* create compressed buffer with content size missing */ + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) ); + CHECKPLUS(r, ZSTD_compress2(cctx, + compressedBuffer, compressedBufferSize, + CNBuffer, CNBuffSize), + cSize=r ); + ZSTD_freeCCtx(cctx); + } + { /* ensure frame content size is missing */ + ZSTD_frameHeader zfh; + size_t const ret = ZSTD_getFrameHeader(&zfh, compressedBuffer, compressedBufferSize); + if (ret != 0 || zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error; + } + { /* ensure CNBuffSize <= decompressBound */ + unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, compressedBufferSize); + if (CNBuffSize > bound) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3d : check CCtx size after compressing empty input : ", testNb++); { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); size_t const r = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 19); @@ -828,6 +866,59 @@ static int basicUnitTests(U32 seed, double compressibility) ZSTDMT_freeCCtx(mtctx); } + DISPLAYLEVEL(3, "test%3i : compress -T2 with/without literals compression : ", testNb++) + { ZSTD_CCtx* cctx = ZSTD_createCCtx(); + size_t cSize1, cSize2; + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2) ); + cSize1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); + CHECK(cSize1); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_lcm_uncompressed) ); + cSize2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); + CHECK(cSize2); + CHECK_LT(cSize1, cSize2); + ZSTD_freeCCtx(cctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Multithreaded ZSTD_compress2() with rsyncable : ", testNb++) + { ZSTD_CCtx* cctx = ZSTD_createCCtx(); + /* Set rsyncable and don't give the ZSTD_compressBound(CNBuffSize) so + * ZSTDMT is forced to not take the shortcut. + */ + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1) ); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_rsyncable, 1) ); + CHECK( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize - 1, CNBuffer, CNBuffSize) ); + ZSTD_freeCCtx(cctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : setting multithreaded parameters : ", testNb++) + { ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); + int value; + /* Check that the overlap log and job size are unset. */ + CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) ); + CHECK_EQ(value, 0); + CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) ); + CHECK_EQ(value, 0); + /* Set and check the overlap log and job size. */ + CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) ); + CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, 2 MB) ); + CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) ); + CHECK_EQ(value, 5); + CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) ); + CHECK_EQ(value, 2 MB); + /* Set the number of workers and check the overlap log and job size. */ + CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) ); + CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) ); + CHECK_EQ(value, 5); + CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) ); + CHECK_EQ(value, 2 MB); + ZSTD_freeCCtxParams(params); + + } + DISPLAYLEVEL(3, "OK \n"); /* Simple API multiframe test */ DISPLAYLEVEL(3, "test%3i : compress multiple frames : ", testNb++); @@ -859,6 +950,11 @@ static int basicUnitTests(U32 seed, double compressibility) if (r != CNBuffSize / 2) goto _output_error; } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : get tight decompressed bound of multiple frames : ", testNb++); + { unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, cSize); + if (bound != CNBuffSize / 2) goto _output_error; } + DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : decompress multiple frames : ", testNb++); { CHECK_V(r, ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize)); if (r != CNBuffSize / 2) goto _output_error; } @@ -1203,9 +1299,13 @@ static int basicUnitTests(U32 seed, double compressibility) { size_t ret; MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY); + /* Either operation is allowed to fail, but one must fail. */ ret = ZSTD_CCtx_loadDictionary_advanced( cctx, (const char*)dictBuffer+2, dictSize-2, ZSTD_dlm_byRef, ZSTD_dct_auto); - if (!ZSTD_isError(ret)) goto _output_error; + if (!ZSTD_isError(ret)) { + ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)); + if (!ZSTD_isError(ret)) goto _output_error; + } } DISPLAYLEVEL(3, "OK \n"); @@ -1216,6 +1316,187 @@ static int basicUnitTests(U32 seed, double compressibility) ret = ZSTD_CCtx_loadDictionary_advanced( cctx, (const char*)dictBuffer+2, dictSize-2, ZSTD_dlm_byRef, ZSTD_dct_rawContent); if (ZSTD_isError(ret)) goto _output_error; + ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)); + if (ZSTD_isError(ret)) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_CCtx_refCDict() then set parameters : ", testNb++); + { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 1); + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 )); + CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 )); + ZSTD_freeCDict(cdict); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Loading dictionary before setting parameters is the same as loading after : ", testNb++); + { + size_t size1, size2; + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) ); + CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); + size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); + if (ZSTD_isError(size1)) goto _output_error; + + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) ); + size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); + if (ZSTD_isError(size2)) goto _output_error; + + if (size1 != size2) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the prefix : ", testNb++); + { + CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); + CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); + CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the cdict : ", testNb++); + { + ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); + CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); + CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); + CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); + ZSTD_freeCDict(cdict); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the prefix : ", testNb++); + { + ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); + CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); + CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); + CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); + ZSTD_freeCDict(cdict); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the dictionary : ", testNb++); + { + ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); + CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); + CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); + CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); + ZSTD_freeCDict(cdict); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the dictionary : ", testNb++); + { + CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); + CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); + CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the cdict : ", testNb++); + { + ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); + CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); + CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); + CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); + ZSTD_freeCDict(cdict); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Loaded dictionary persists across reset session : ", testNb++); + { + size_t size1, size2; + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); + size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); + if (ZSTD_isError(size1)) goto _output_error; + + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); + size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); + if (ZSTD_isError(size2)) goto _output_error; + + if (size1 != size2) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Loaded dictionary is cleared after resetting parameters : ", testNb++); + { + size_t size1, size2; + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); + size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); + if (ZSTD_isError(size1)) goto _output_error; + + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); + if (ZSTD_isError(size2)) goto _output_error; + + if (size1 == size2) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, dictBuffer, dictSize) ); + cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); + CHECK_Z(cSize); + DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with dictionary : ", testNb++); + { + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + size_t ret; + /* We should fail to decompress without a dictionary. */ + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); + ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); + if (!ZSTD_isError(ret)) goto _output_error; + /* We should succeed to decompress with the dictionary. */ + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); + CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictSize) ); + CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); + /* The dictionary should presist across calls. */ + CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); + /* When we reset the context the dictionary is cleared. */ + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); + ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); + if (!ZSTD_isError(ret)) goto _output_error; + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with ddict : ", testNb++); + { + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictSize); + size_t ret; + /* We should succeed to decompress with the ddict. */ + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); + CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) ); + CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); + /* The ddict should presist across calls. */ + CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); + /* When we reset the context the ddict is cleared. */ + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); + ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); + if (!ZSTD_isError(ret)) goto _output_error; + ZSTD_freeDCtx(dctx); + ZSTD_freeDDict(ddict); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++); + { + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + size_t ret; + /* We should succeed to decompress with the prefix. */ + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); + CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictBuffer, dictSize, ZSTD_dct_auto) ); + CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); + /* The prefix should be cleared after the first compression. */ + ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); + if (!ZSTD_isError(ret)) goto _output_error; + ZSTD_freeDCtx(dctx); } DISPLAYLEVEL(3, "OK \n"); diff --git a/tests/legacy.c b/tests/legacy.c index e1cf82f2f9d8..eb329203833f 100644 --- a/tests/legacy.c +++ b/tests/legacy.c @@ -16,10 +16,11 @@ /*=========================================== * Dependencies *==========================================*/ -#include <stddef.h> /* size_t */ -#include <stdlib.h> /* malloc, free */ -#include <stdio.h> /* fprintf */ -#include <string.h> /* strlen */ +#include <stddef.h> /* size_t */ +#include <stdlib.h> /* malloc, free */ +#include <stdio.h> /* fprintf */ +#include <string.h> /* strlen */ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_decompressBound */ #include "zstd.h" #include "zstd_errors.h" @@ -74,6 +75,7 @@ static int testSimpleAPI(void) static int testStreamingAPI(void) { + int error_code = 0; size_t const outBuffSize = ZSTD_DStreamOutSize(); char* const outBuff = malloc(outBuffSize); ZSTD_DStream* const stream = ZSTD_createDStream(); @@ -87,6 +89,7 @@ static int testStreamingAPI(void) } if (stream == NULL) { DISPLAY("ERROR: Could not create dstream\n"); + free(outBuff); return 1; } @@ -96,13 +99,15 @@ static int testStreamingAPI(void) size_t const ret = ZSTD_initDStream(stream); if (ZSTD_isError(ret)) { DISPLAY("ERROR: ZSTD_initDStream: %s\n", ZSTD_getErrorName(ret)); - return 1; + error_code = 1; + break; } } { size_t const ret = ZSTD_decompressStream(stream, &output, &input); if (ZSTD_isError(ret)) { DISPLAY("ERROR: ZSTD_decompressStream: %s\n", ZSTD_getErrorName(ret)); - return 1; + error_code = 1; + break; } if (ret == 0) { @@ -111,7 +116,8 @@ static int testStreamingAPI(void) if (memcmp(outBuff, EXPECTED + outputPos, output.pos) != 0) { DISPLAY("ERROR: Wrong decoded output produced\n"); - return 1; + error_code = 1; + break; } outputPos += output.pos; if (input.pos == input.size && output.pos < output.size) { @@ -121,7 +127,34 @@ static int testStreamingAPI(void) free(outBuff); ZSTD_freeDStream(stream); - DISPLAY("Streaming API OK\n"); + if (error_code == 0) DISPLAY("Streaming API OK\n"); + return error_code; +} + +static int testFrameDecoding(void) +{ + if (strlen(EXPECTED) > ZSTD_decompressBound(COMPRESSED, COMPRESSED_SIZE)) { + DISPLAY("ERROR: ZSTD_decompressBound: decompressed bound too small\n"); + return 1; + } + { const char* ip = COMPRESSED; + size_t remainingSize = COMPRESSED_SIZE; + while (1) { + size_t frameSize = ZSTD_findFrameCompressedSize(ip, remainingSize); + if (ZSTD_isError(frameSize)) { + DISPLAY("ERROR: ZSTD_findFrameCompressedSize: %s\n", ZSTD_getErrorName(frameSize)); + return 1; + } + if (frameSize > remainingSize) { + DISPLAY("ERROR: ZSTD_findFrameCompressedSize: expected frameSize to align with src buffer"); + return 1; + } + ip += frameSize; + remainingSize -= frameSize; + if (remainingSize == 0) break; + } + } + DISPLAY("Frame Decoding OK\n"); return 0; } @@ -131,6 +164,8 @@ int main(void) if (ret) return ret; } { int const ret = testStreamingAPI(); if (ret) return ret; } + { int const ret = testFrameDecoding(); + if (ret) return ret; } DISPLAY("OK\n"); return 0; diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 415551b9578d..75c179a0ba0e 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -19,6 +19,7 @@ #include <math.h> /* log */ #include <assert.h> +#include "timefn.h" /* SEC_TO_MICRO, UTIL_time_t, UTIL_clockSpanMicro, UTIL_clockSpanNano, UTIL_getTime */ #include "mem.h" #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_estimateCCtxSize */ #include "zstd.h" @@ -141,7 +142,7 @@ static const char* g_shortParamNames[NUM_PARAMS] = /* maps value from { 0 to rangetable[param] - 1 } to valid paramvalues */ static U32 rangeMap(varInds_t param, int ind) { - ind = MAX(MIN(ind, (int)rangetable[param] - 1), 0); + U32 const uind = (U32)MAX(MIN(ind, (int)rangetable[param] - 1), 0); switch(param) { case wlog_ind: /* using default: triggers -Wswitch-enum */ case clog_ind: @@ -149,11 +150,11 @@ static U32 rangeMap(varInds_t param, int ind) case slog_ind: case mml_ind: case strt_ind: - return mintable[param] + ind; + return mintable[param] + uind; case tlen_ind: - return tlen_table[ind]; + return tlen_table[uind]; case fadt_ind: /* 0, 1, 2 -> -1, 0, 1 */ - return ind - 1; + return uind - 1; case NUM_PARAMS: default:; } @@ -173,7 +174,7 @@ static int invRangeMap(varInds_t param, U32 value) case slog_ind: case mml_ind: case strt_ind: - return value - mintable[param]; + return (int)(value - mintable[param]); case tlen_ind: /* bin search */ { int lo = 0; @@ -493,13 +494,15 @@ static void paramVariation(paramValues_t* ptr, memoTable_t* mtAll, const U32 nbChanges) { paramValues_t p; - U32 validated = 0; + int validated = 0; while (!validated) { U32 i; p = *ptr; for (i = 0 ; i < nbChanges ; i++) { const U32 changeID = (U32)FUZ_rand(&g_rand) % (mtAll[p.vals[strt_ind]].varLen << 1); - paramVaryOnce(mtAll[p.vals[strt_ind]].varArray[changeID >> 1], ((changeID & 1) << 1) - 1, &p); + paramVaryOnce(mtAll[p.vals[strt_ind]].varArray[changeID >> 1], + (int)((changeID & 1) << 1) - 1, + &p); } validated = paramValid(p); } @@ -511,7 +514,7 @@ static paramValues_t randomParams(void) { varInds_t v; paramValues_t p; for(v = 0; v < NUM_PARAMS; v++) { - p.vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]); + p.vals[v] = rangeMap(v, (int)(FUZ_rand(&g_rand) % rangetable[v])); } return p; } @@ -1232,7 +1235,7 @@ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, const size return 0; } -/* allocates buffer's arguments. returns success / failuere */ +/* allocates buffer's arguments. returns success / failure */ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable, size_t nbFiles) { size_t pos = 0; @@ -1505,7 +1508,7 @@ createMemoTableArray(const paramValues_t p, } /* Sets pc to random unmeasured set of parameters */ -/* specifiy strategy */ +/* specify strategy */ static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTableArray, const ZSTD_strategy st) { size_t j; @@ -1570,7 +1573,7 @@ BMK_benchMemInvertible( buffers_t buf, contexts_t ctx, display_params_tested(*comprParams); memset(&bResult, 0, sizeof(bResult)); - /* warmimg up memory */ + /* warming up memory */ for (i = 0; i < buf.nbBlocks; i++) { if (mode != BMK_decodeOnly) { RDG_genBuffer(dstPtrs[i], dstCapacities[i], 0.10, 0.50, 1); @@ -1638,7 +1641,7 @@ BMK_benchMemInvertible( buffers_t buf, contexts_t ctx, return bOut; } { BMK_runTime_t const rResult = BMK_extract_runTime(cOutcome); - bResult.cSpeed = (srcSize * TIMELOOP_NANOSEC) / rResult.nanoSecPerRun; + bResult.cSpeed = (unsigned long long)((double)srcSize * TIMELOOP_NANOSEC / rResult.nanoSecPerRun); bResult.cSize = rResult.sumOfReturn; } compressionCompleted = BMK_isCompleted_TimedFn(timeStateCompress); @@ -1656,7 +1659,7 @@ BMK_benchMemInvertible( buffers_t buf, contexts_t ctx, return bOut; } { BMK_runTime_t const rResult = BMK_extract_runTime(dOutcome); - bResult.dSpeed = (srcSize * TIMELOOP_NANOSEC) / rResult.nanoSecPerRun; + bResult.dSpeed = (unsigned long long)((double)srcSize * TIMELOOP_NANOSEC / rResult.nanoSecPerRun); } decompressionCompleted = BMK_isCompleted_TimedFn(timeStateDecompress); } @@ -1989,7 +1992,7 @@ BMK_selectRandomStart( FILE* f, /* BMK_generate_cLevelTable() : * test a large number of configurations - * and distribute them accross compression levels according to speed conditions. + * and distribute them across compression levels according to speed conditions. * display and save all intermediate results into rfName = "grillResults.txt". * the function automatically stops after g_timeLimit_s. * this function cannot error, it directly exit() in case of problem. @@ -2199,7 +2202,9 @@ static winnerInfo_t climbOnce(const constraint_t target, for (offset = -1; offset <= 1; offset += 2) { CHECKTIME(winnerInfo); candidateInfo.params = cparam; - paramVaryOnce(mtAll[cparam.vals[strt_ind]].varArray[i], offset, &candidateInfo.params); + paramVaryOnce(mtAll[cparam.vals[strt_ind]].varArray[i], + offset, + &candidateInfo.params); if(paramValid(candidateInfo.params)) { int res; @@ -2265,7 +2270,7 @@ static winnerInfo_t climbOnce(const constraint_t target, /* Optimizes for a fixed strategy */ -/* flexible parameters: iterations of failed climbing (or if we do non-random, maybe this is when everything is close to visitied) +/* flexible parameters: iterations of failed climbing (or if we do non-random, maybe this is when everything is close to visited) weight more on visit for bad results, less on good results/more on later results / ones with more failures. allocate memoTable here. */ @@ -2351,7 +2356,7 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio) */ -static int g_maxTries = 5; +static unsigned g_maxTries = 5; #define TRY_DECAY 1 static int @@ -2561,7 +2566,7 @@ _cleanUp: * @return 0 and doesn't modify *stringPtr otherwise. * from zstdcli.c */ -static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) +static int longCommandWArg(const char** stringPtr, const char* longCommand) { size_t const comSize = strlen(longCommand); int const result = !strncmp(*stringPtr, longCommand, comSize); @@ -2588,7 +2593,10 @@ static unsigned readU32FromChar(const char** stringPtr) while ((**stringPtr >='0') && (**stringPtr <='9')) { unsigned const max = (((unsigned)(-1)) / 10) - 1; if (result > max) errorOut(errorMsg); - result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + result *= 10; + assert(**stringPtr >= '0'); + result += (unsigned)(**stringPtr - '0'); + (*stringPtr)++ ; } if ((**stringPtr=='K') || (**stringPtr=='M')) { unsigned const maxK = ((unsigned)(-1)) >> 10; @@ -2726,7 +2734,7 @@ int main(int argc, const char** argv) PARSE_SUB_ARGS("strict=", "stc=", g_strictness); PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries); PARSE_SUB_ARGS("memoLimitLog=", "memLog=", memoTableLog); - if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelOpt = readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelOpt = (int)readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "speedForRatio=") || longCommandWArg(&argument, "speedRatio=")) { g_ratioMultiplier = readDoubleFromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } DISPLAY("invalid optimization parameter \n"); @@ -2743,7 +2751,7 @@ int main(int argc, const char** argv) g_singleRun = 1; for ( ; ;) { if(parse_params(&argument, &g_params)) { if(argument[0] == ',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelRun = readU32FromChar(&argument); g_params = emptyParams(); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelRun = (int)readU32FromChar(&argument); g_params = emptyParams(); if (argument[0]==',') { argument++; continue; } else break; } DISPLAY("invalid compression parameter \n"); return 1; @@ -2855,7 +2863,7 @@ int main(int argc, const char** argv) continue; case 'L': { argument++; - cLevelRun = readU32FromChar(&argument); + cLevelRun = (int)readU32FromChar(&argument); g_params = emptyParams(); continue; } @@ -2944,7 +2952,8 @@ int main(int argc, const char** argv) } } else { if (g_optimizer) { - result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, cLevelOpt, cLevelRun, memoTableLog); + assert(filenamesStart < argc); + result = optimizeForSize(argv+filenamesStart, (size_t)(argc-filenamesStart), dictFileName, target, paramTarget, cLevelOpt, cLevelRun, memoTableLog); } else { result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName, cLevelRun); } diff --git a/tests/playTests.sh b/tests/playTests.sh index ef4861b52917..bce2710159ed 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -49,7 +49,7 @@ fileRoundTripTest() { } truncateLastByte() { - dd bs=1 count=$(($(wc -c < "$1") - 1)) if="$1" status=none + dd bs=1 count=$(($(wc -c < "$1") - 1)) if="$1" } UNAME=$(uname) @@ -200,6 +200,15 @@ $ZSTD tmp -fo tmp && die "zstd compression overwrote the input file" $ZSTD tmp.zst -dfo tmp.zst && die "zstd decompression overwrote the input file" $ECHO "test: detect that input file does not exist" $ZSTD nothere && die "zstd hasn't detected that input file does not exist" +$ECHO "test: --[no-]compress-literals" +$ZSTD tmp -c --no-compress-literals -1 | $ZSTD -t +$ZSTD tmp -c --no-compress-literals --fast=1 | $ZSTD -t +$ZSTD tmp -c --no-compress-literals -19 | $ZSTD -t +$ZSTD tmp -c --compress-literals -1 | $ZSTD -t +$ZSTD tmp -c --compress-literals --fast=1 | $ZSTD -t +$ZSTD tmp -c --compress-literals -19 | $ZSTD -t +$ZSTD -b --fast=1 -i1e1 tmp --compress-literals +$ZSTD -b --fast=1 -i1e1 tmp --no-compress-literals $ECHO "test : file removal" $ZSTD -f --rm tmp @@ -314,18 +323,28 @@ $ECHO foo | $ZSTD > /dev/full && die "write error not detected!" $ECHO "$ECHO foo | $ZSTD | $ZSTD -d > /dev/full" $ECHO foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!" +fi + + +if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] ; then $ECHO "\n===> symbolic link test " -rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst +rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst $ECHO "hello world" > hello.tmp ln -s hello.tmp world.tmp -$ZSTD world.tmp hello.tmp +ln -s hello.tmp world2.tmp +$ZSTD world.tmp hello.tmp || true test -f hello.tmp.zst # regular file should have been compressed! test ! -f world.tmp.zst # symbolic link should not have been compressed! +$ZSTD world.tmp || true +test ! -f world.tmp.zst # symbolic link should not have been compressed! +$ZSTD world.tmp world2.tmp || true +test ! -f world.tmp.zst # symbolic link should not have been compressed! +test ! -f world2.tmp.zst # symbolic link should not have been compressed! $ZSTD world.tmp hello.tmp -f test -f world.tmp.zst # symbolic link should have been compressed with --force -rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst +rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst fi @@ -391,6 +410,8 @@ $ECHO "- Create first dictionary " TESTFILE=../programs/zstdcli.c $ZSTD --train *.c ../programs/*.c -o tmpDict cp $TESTFILE tmp +$ECHO "- Test dictionary compression with tmpDict as an input file and dictionary" +$ZSTD -f tmpDict -D tmpDict && die "compression error not detected!" $ECHO "- Dictionary compression roundtrip" $ZSTD -f tmp -D tmpDict $ZSTD -d tmp.zst -D tmpDict -fo result @@ -809,6 +830,19 @@ $ZSTD --list tmp* && die "-l must fail on non-zstd file" $ZSTD -lv tmp1* && die "-l must fail on non-zstd file" $ZSTD --list -v tmp2 tmp12.zst && die "-l must fail on non-zstd file" +$ECHO "test : detect truncated compressed file " +TEST_DATA_FILE=truncatable-input.txt +FULL_COMPRESSED_FILE=${TEST_DATA_FILE}.zst +TRUNCATED_COMPRESSED_FILE=truncated-input.txt.zst +./datagen -g50000 > $TEST_DATA_FILE +$ZSTD -f $TEST_DATA_FILE -o $FULL_COMPRESSED_FILE +dd bs=1 count=100 if=$FULL_COMPRESSED_FILE of=$TRUNCATED_COMPRESSED_FILE +$ZSTD --list $TRUNCATED_COMPRESSED_FILE && die "-l must fail on truncated file" + +rm $TEST_DATA_FILE +rm $FULL_COMPRESSED_FILE +rm $TRUNCATED_COMPRESSED_FILE + $ECHO "\n===> zstd --list/-l errors when presented with stdin / no files" $ZSTD -l && die "-l must fail on empty list of files" $ZSTD -l - && die "-l does not work on stdin" diff --git a/tests/poolTests.c b/tests/poolTests.c index 8b9a4700229d..272e961d3800 100644 --- a/tests/poolTests.c +++ b/tests/poolTests.c @@ -12,6 +12,7 @@ #include "pool.h" #include "threading.h" #include "util.h" +#include "timefn.h" #include <stddef.h> #include <stdio.h> @@ -25,25 +26,27 @@ #define ASSERT_EQ(lhs, rhs) ASSERT_TRUE((lhs) == (rhs)) struct data { - pthread_mutex_t mutex; + ZSTD_pthread_mutex_t mutex; unsigned data[16]; size_t i; }; -static void fn(void *opaque) { +static void fn(void *opaque) +{ struct data *data = (struct data *)opaque; ZSTD_pthread_mutex_lock(&data->mutex); - data->data[data->i] = data->i; + data->data[data->i] = (unsigned)(data->i); ++data->i; ZSTD_pthread_mutex_unlock(&data->mutex); } -static int testOrder(size_t numThreads, size_t queueSize) { +static int testOrder(size_t numThreads, size_t queueSize) +{ struct data data; - POOL_ctx *ctx = POOL_create(numThreads, queueSize); + POOL_ctx* const ctx = POOL_create(numThreads, queueSize); ASSERT_TRUE(ctx); data.i = 0; - ZSTD_pthread_mutex_init(&data.mutex, NULL); + (void)ZSTD_pthread_mutex_init(&data.mutex, NULL); { size_t i; for (i = 0; i < 16; ++i) { POOL_add(ctx, &fn, &data); @@ -71,7 +74,7 @@ static void waitFn(void *opaque) { /* Tests for deadlock */ static int testWait(size_t numThreads, size_t queueSize) { struct data data; - POOL_ctx *ctx = POOL_create(numThreads, queueSize); + POOL_ctx* const ctx = POOL_create(numThreads, queueSize); ASSERT_TRUE(ctx); { size_t i; for (i = 0; i < 16; ++i) { @@ -93,7 +96,7 @@ typedef struct { } poolTest_t; static void waitLongFn(void *opaque) { - poolTest_t* test = (poolTest_t*) opaque; + poolTest_t* const test = (poolTest_t*) opaque; UTIL_sleepMilli(10); ZSTD_pthread_mutex_lock(&test->mut); test->val = test->val + 1; diff --git a/tests/regression/config.c b/tests/regression/config.c index 8a6103b2929e..b82482f46123 100644 --- a/tests/regression/config.c +++ b/tests/regression/config.c @@ -90,6 +90,17 @@ static config_t mt_ldm = { .param_values = PARAM_VALUES(mt_ldm_param_values), }; +static param_value_t mt_advanced_param_values[] = { + {.param = ZSTD_c_nbWorkers, .value = 2}, + {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_lcm_uncompressed}, +}; + +static config_t mt_advanced = { + .name = "multithreaded with advanced params", + .cli_args = "-T2 --no-compress-literals", + .param_values = PARAM_VALUES(mt_advanced_param_values), +}; + static param_value_t const small_wlog_param_values[] = { {.param = ZSTD_c_windowLog, .value = 10}, }; @@ -122,6 +133,39 @@ static config_t small_clog = { .param_values = PARAM_VALUES(small_clog_param_values), }; +static param_value_t const uncompressed_literals_param_values[] = { + {.param = ZSTD_c_compressionLevel, .value = 3}, + {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_lcm_uncompressed}, +}; + +static config_t uncompressed_literals = { + .name = "uncompressed literals", + .cli_args = "-3 --no-compress-literals", + .param_values = PARAM_VALUES(uncompressed_literals_param_values), +}; + +static param_value_t const uncompressed_literals_opt_param_values[] = { + {.param = ZSTD_c_compressionLevel, .value = 19}, + {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_lcm_uncompressed}, +}; + +static config_t uncompressed_literals_opt = { + .name = "uncompressed literals optimal", + .cli_args = "-19 --no-compress-literals", + .param_values = PARAM_VALUES(uncompressed_literals_opt_param_values), +}; + +static param_value_t const huffman_literals_param_values[] = { + {.param = ZSTD_c_compressionLevel, .value = -1}, + {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_lcm_huffman}, +}; + +static config_t huffman_literals = { + .name = "huffman literals", + .cli_args = "--fast=1 --compress-literals", + .param_values = PARAM_VALUES(huffman_literals_param_values), +}; + static param_value_t const explicit_params_param_values[] = { {.param = ZSTD_c_checksumFlag, .value = 1}, {.param = ZSTD_c_contentSizeFlag, .value = 0}, @@ -155,6 +199,10 @@ static config_t const* g_configs[] = { &small_hlog, &small_clog, &explicit_params, + &uncompressed_literals, + &uncompressed_literals_opt, + &huffman_literals, + &mt_advanced, NULL, }; diff --git a/tests/regression/method.c b/tests/regression/method.c index 51a0fb9d14f0..1e84021c3ef3 100644 --- a/tests/regression/method.c +++ b/tests/regression/method.c @@ -175,8 +175,8 @@ static result_t compress_cctx_compress( state->compressed.capacity, input.data, input.size, - state->dictionary.data, - state->dictionary.size, + config->use_dictionary ? state->dictionary.data : NULL, + config->use_dictionary ? state->dictionary.size : 0, params); else if (config->use_dictionary) state->compressed.size = ZSTD_compress_usingDict( @@ -432,77 +432,158 @@ out: return result; } -static result_t old_streaming_compress( - method_state_t* base, - config_t const* config) { - buffer_state_t* state = container_of(base, buffer_state_t, base); - - if (buffer_state_bad(state, config)) - return result_error(result_error_system_error); - - int const level = config_get_level(config); - if (level == CONFIG_NO_LEVEL) - return result_error(result_error_skip); - - ZSTD_CStream* zcs = ZSTD_createCStream(); - result_t result; - if (zcs == NULL) { - result = result_error(result_error_compression_error); - goto out; - } +static int init_cstream( + buffer_state_t* state, + ZSTD_CStream* zcs, + config_t const* config, + int const advanced, + ZSTD_CDict** cdict) +{ size_t zret; - if (config->use_dictionary) { - zret = ZSTD_initCStream_usingDict( - zcs, state->dictionary.data, state->dictionary.size, level); + if (advanced) { + ZSTD_parameters const params = config_get_zstd_params(config, 0, 0); + ZSTD_CDict* dict = NULL; + if (cdict) { + *cdict = ZSTD_createCDict_advanced( + state->dictionary.data, + state->dictionary.size, + ZSTD_dlm_byRef, + ZSTD_dct_auto, + params.cParams, + ZSTD_defaultCMem); + if (!*cdict) { + return 1; + } + zret = ZSTD_initCStream_usingCDict_advanced( + zcs, *cdict, params.fParams, ZSTD_CONTENTSIZE_UNKNOWN); + } else { + zret = ZSTD_initCStream_advanced( + zcs, + state->dictionary.data, + state->dictionary.size, + params, + ZSTD_CONTENTSIZE_UNKNOWN); + } } else { - zret = ZSTD_initCStream(zcs, level); + int const level = config_get_level(config); + if (cdict) { + *cdict = ZSTD_createCDict( + state->dictionary.data, + state->dictionary.size, + level); + if (!*cdict) { + return 1; + } + zret = ZSTD_initCStream_usingCDict(zcs, *cdict); + } else if (config->use_dictionary) { + zret = ZSTD_initCStream_usingDict( + zcs, state->dictionary.data, state->dictionary.size, level); + } else { + zret = ZSTD_initCStream(zcs, level); + } } if (ZSTD_isError(zret)) { - result = result_error(result_error_compression_error); - goto out; + return 1; } + return 0; +} - result_data_t data = {.total_size = 0}; - for (size_t i = 0; i < state->inputs.size; ++i) { - data_buffer_t input = state->inputs.buffers[i]; - zret = ZSTD_resetCStream( - zcs, - config->no_pledged_src_size ? ZSTD_CONTENTSIZE_UNKNOWN - : input.size); - if (ZSTD_isError(zret)) { - result = result_error(result_error_compression_error); - goto out; - } +static result_t old_streaming_compress_internal( + method_state_t* base, + config_t const* config, + int const advanced, + int const cdict) { + buffer_state_t* state = container_of(base, buffer_state_t, base); + + if (buffer_state_bad(state, config)) + return result_error(result_error_system_error); + + + ZSTD_CStream* zcs = ZSTD_createCStream(); + ZSTD_CDict* cd = NULL; + result_t result; + if (zcs == NULL) { + result = result_error(result_error_compression_error); + goto out; + } + if (init_cstream(state, zcs, config, advanced, cdict ? &cd : NULL)) { + result = result_error(result_error_compression_error); + goto out; + } + + result_data_t data = {.total_size = 0}; + for (size_t i = 0; i < state->inputs.size; ++i) { + data_buffer_t input = state->inputs.buffers[i]; + size_t zret = ZSTD_resetCStream( + zcs, + config->no_pledged_src_size ? ZSTD_CONTENTSIZE_UNKNOWN : input.size); + if (ZSTD_isError(zret)) { + result = result_error(result_error_compression_error); + goto out; + } - while (input.size > 0) { - ZSTD_inBuffer in = {input.data, MIN(input.size, 4096)}; - input.data += in.size; - input.size -= in.size; - ZSTD_EndDirective const op = - input.size > 0 ? ZSTD_e_continue : ZSTD_e_end; - zret = 0; - while (in.pos < in.size || (op == ZSTD_e_end && zret != 0)) { - ZSTD_outBuffer out = {state->compressed.data, - MIN(state->compressed.capacity, 1024)}; - if (op == ZSTD_e_continue || in.pos < in.size) - zret = ZSTD_compressStream(zcs, &out, &in); - else - zret = ZSTD_endStream(zcs, &out); - if (ZSTD_isError(zret)) { - result = result_error(result_error_compression_error); - goto out; - } - data.total_size += out.pos; - } + while (input.size > 0) { + ZSTD_inBuffer in = {input.data, MIN(input.size, 4096)}; + input.data += in.size; + input.size -= in.size; + ZSTD_EndDirective const op = + input.size > 0 ? ZSTD_e_continue : ZSTD_e_end; + zret = 0; + while (in.pos < in.size || (op == ZSTD_e_end && zret != 0)) { + ZSTD_outBuffer out = {state->compressed.data, + MIN(state->compressed.capacity, 1024)}; + if (op == ZSTD_e_continue || in.pos < in.size) + zret = ZSTD_compressStream(zcs, &out, &in); + else + zret = ZSTD_endStream(zcs, &out); + if (ZSTD_isError(zret)) { + result = result_error(result_error_compression_error); + goto out; } + data.total_size += out.pos; + } } + } - result = result_data(data); + result = result_data(data); out: ZSTD_freeCStream(zcs); + ZSTD_freeCDict(cd); return result; } +static result_t old_streaming_compress( + method_state_t* base, + config_t const* config) +{ + return old_streaming_compress_internal( + base, config, /* advanced */ 0, /* cdict */ 0); +} + +static result_t old_streaming_compress_advanced( + method_state_t* base, + config_t const* config) +{ + return old_streaming_compress_internal( + base, config, /* advanced */ 1, /* cdict */ 0); +} + +static result_t old_streaming_compress_cdict( + method_state_t* base, + config_t const* config) +{ + return old_streaming_compress_internal( + base, config, /* advanced */ 0, /* cdict */ 1); +} + +static result_t old_streaming_compress_cdict_advanced( + method_state_t* base, + config_t const* config) +{ + return old_streaming_compress_internal( + base, config, /* advanced */ 1, /* cdict */ 1); +} + method_t const simple = { .name = "compress simple", .create = buffer_state_create, @@ -545,6 +626,27 @@ method_t const old_streaming = { .destroy = buffer_state_destroy, }; +method_t const old_streaming_advanced = { + .name = "old streaming advanced", + .create = buffer_state_create, + .compress = old_streaming_compress, + .destroy = buffer_state_destroy, +}; + +method_t const old_streaming_cdict = { + .name = "old streaming cdcit", + .create = buffer_state_create, + .compress = old_streaming_compress, + .destroy = buffer_state_destroy, +}; + +method_t const old_streaming_advanced_cdict = { + .name = "old streaming advanced cdict", + .create = buffer_state_create, + .compress = old_streaming_compress, + .destroy = buffer_state_destroy, +}; + method_t const cli = { .name = "zstdcli", .create = method_state_create, @@ -560,6 +662,9 @@ static method_t const* g_methods[] = { &advanced_one_pass_small_out, &advanced_streaming, &old_streaming, + &old_streaming_advanced, + &old_streaming_cdict, + &old_streaming_advanced_cdict, NULL, }; diff --git a/tests/regression/results.csv b/tests/regression/results.csv index a62178d6ed0c..7ac94f207a6c 100644 --- a/tests/regression/results.csv +++ b/tests/regression/results.csv @@ -1,448 +1,816 @@ -Data, Config, Method, Total compressed size -silesia.tar, level -5, compress simple, 7160438 -silesia.tar, level -3, compress simple, 6789024 -silesia.tar, level -1, compress simple, 6195462 -silesia.tar, level 0, compress simple, 4875008 -silesia.tar, level 1, compress simple, 5339697 -silesia.tar, level 3, compress simple, 4875008 -silesia.tar, level 4, compress simple, 4813507 -silesia.tar, level 5, compress simple, 4722235 -silesia.tar, level 6, compress simple, 4672194 -silesia.tar, level 7, compress simple, 4606658 -silesia.tar, level 9, compress simple, 4554098 -silesia.tar, level 13, compress simple, 4491702 -silesia.tar, level 16, compress simple, 4381277 -silesia.tar, level 19, compress simple, 4281514 -silesia, level -5, compress cctx, 7152294 -silesia, level -3, compress cctx, 6789969 -silesia, level -1, compress cctx, 6191548 -silesia, level 0, compress cctx, 4862377 -silesia, level 1, compress cctx, 5318036 -silesia, level 3, compress cctx, 4862377 -silesia, level 4, compress cctx, 4800629 -silesia, level 5, compress cctx, 4710178 -silesia, level 6, compress cctx, 4659996 -silesia, level 7, compress cctx, 4596234 -silesia, level 9, compress cctx, 4543862 -silesia, level 13, compress cctx, 4482073 -silesia, level 16, compress cctx, 4377391 -silesia, level 19, compress cctx, 4293262 -silesia, long distance mode, compress cctx, 4862377 -silesia, multithreaded, compress cctx, 4862377 -silesia, multithreaded long distance mode, compress cctx, 4862377 -silesia, small window log, compress cctx, 7115734 -silesia, small hash log, compress cctx, 6554898 -silesia, small chain log, compress cctx, 4931093 -silesia, explicit params, compress cctx, 4813352 -github, level -5, compress cctx, 232744 -github, level -5 with dict, compress cctx, 47294 -github, level -3, compress cctx, 220611 -github, level -3 with dict, compress cctx, 48047 -github, level -1, compress cctx, 176575 -github, level -1 with dict, compress cctx, 43527 -github, level 0, compress cctx, 136397 -github, level 0 with dict, compress cctx, 41536 -github, level 1, compress cctx, 143457 -github, level 1 with dict, compress cctx, 42157 -github, level 3, compress cctx, 136397 -github, level 3 with dict, compress cctx, 41536 -github, level 4, compress cctx, 136144 -github, level 4 with dict, compress cctx, 41721 -github, level 5, compress cctx, 135106 -github, level 5 with dict, compress cctx, 38934 -github, level 6, compress cctx, 135108 -github, level 6 with dict, compress cctx, 38628 -github, level 7, compress cctx, 135108 -github, level 7 with dict, compress cctx, 38741 -github, level 9, compress cctx, 135108 -github, level 9 with dict, compress cctx, 39335 -github, level 13, compress cctx, 133717 -github, level 13 with dict, compress cctx, 39923 -github, level 16, compress cctx, 133717 -github, level 16 with dict, compress cctx, 37568 -github, level 19, compress cctx, 133717 -github, level 19 with dict, compress cctx, 37567 -github, long distance mode, compress cctx, decompression error -github, multithreaded, compress cctx, decompression error -github, multithreaded long distance mode, compress cctx, decompression error -github, small window log, compress cctx, decompression error -github, small hash log, compress cctx, decompression error -github, small chain log, compress cctx, decompression error -github, explicit params, compress cctx, decompression error -silesia, level -5, zstdcli, 7152342 -silesia, level -3, zstdcli, 6790021 -silesia, level -1, zstdcli, 6191597 -silesia, level 0, zstdcli, 4862425 -silesia, level 1, zstdcli, 5318084 -silesia, level 3, zstdcli, 4862425 -silesia, level 4, zstdcli, 4800677 -silesia, level 5, zstdcli, 4710226 -silesia, level 6, zstdcli, 4660044 -silesia, level 7, zstdcli, 4596282 -silesia, level 9, zstdcli, 4543910 -silesia, level 13, zstdcli, 4482121 -silesia, level 16, zstdcli, 4377439 -silesia, level 19, zstdcli, 4293310 -silesia, long distance mode, zstdcli, 4853437 -silesia, multithreaded, zstdcli, 4862425 -silesia, multithreaded long distance mode, zstdcli, 4853437 -silesia, small window log, zstdcli, 7126434 -silesia, small hash log, zstdcli, 6554946 -silesia, small chain log, zstdcli, 4931141 -silesia, explicit params, zstdcli, 4815380 -silesia.tar, level -5, zstdcli, 7161160 -silesia.tar, level -3, zstdcli, 6789865 -silesia.tar, level -1, zstdcli, 6196433 -silesia.tar, level 0, zstdcli, 4875136 -silesia.tar, level 1, zstdcli, 5340573 -silesia.tar, level 3, zstdcli, 4875136 -silesia.tar, level 4, zstdcli, 4814531 -silesia.tar, level 5, zstdcli, 4723284 -silesia.tar, level 6, zstdcli, 4673591 -silesia.tar, level 7, zstdcli, 4608342 -silesia.tar, level 9, zstdcli, 4554700 -silesia.tar, level 13, zstdcli, 4491706 -silesia.tar, level 16, zstdcli, 4381281 -silesia.tar, level 19, zstdcli, 4281518 -silesia.tar, no source size, zstdcli, 4875132 -silesia.tar, long distance mode, zstdcli, 4866975 -silesia.tar, multithreaded, zstdcli, 4875136 -silesia.tar, multithreaded long distance mode, zstdcli, 4866975 -silesia.tar, small window log, zstdcli, 7130434 -silesia.tar, small hash log, zstdcli, 6587841 -silesia.tar, small chain log, zstdcli, 4943259 -silesia.tar, explicit params, zstdcli, 4839202 -github, level -5, zstdcli, 234744 -github, level -5 with dict, zstdcli, 48718 -github, level -3, zstdcli, 222611 -github, level -3 with dict, zstdcli, 47395 -github, level -1, zstdcli, 178575 -github, level -1 with dict, zstdcli, 45170 -github, level 0, zstdcli, 138397 -github, level 0 with dict, zstdcli, 43170 -github, level 1, zstdcli, 145457 -github, level 1 with dict, zstdcli, 43682 -github, level 3, zstdcli, 138397 -github, level 3 with dict, zstdcli, 43170 -github, level 4, zstdcli, 138144 -github, level 4 with dict, zstdcli, 43306 -github, level 5, zstdcli, 137106 -github, level 5 with dict, zstdcli, 40938 -github, level 6, zstdcli, 137108 -github, level 6 with dict, zstdcli, 40632 -github, level 7, zstdcli, 137108 -github, level 7 with dict, zstdcli, 40766 -github, level 9, zstdcli, 137108 -github, level 9 with dict, zstdcli, 41326 -github, level 13, zstdcli, 135717 -github, level 13 with dict, zstdcli, 41716 -github, level 16, zstdcli, 135717 -github, level 16 with dict, zstdcli, 39577 -github, level 19, zstdcli, 135717 -github, level 19 with dict, zstdcli, 39576 -github, long distance mode, zstdcli, 138397 -github, multithreaded, zstdcli, 138397 -github, multithreaded long distance mode, zstdcli, 138397 -github, small window log, zstdcli, 138397 -github, small hash log, zstdcli, 137467 -github, small chain log, zstdcli, 138314 -github, explicit params, zstdcli, 136140 -silesia, level -5, advanced one pass, 7152294 -silesia, level -3, advanced one pass, 6789969 -silesia, level -1, advanced one pass, 6191548 -silesia, level 0, advanced one pass, 4862377 -silesia, level 1, advanced one pass, 5318036 -silesia, level 3, advanced one pass, 4862377 -silesia, level 4, advanced one pass, 4800629 -silesia, level 5, advanced one pass, 4710178 -silesia, level 6, advanced one pass, 4659996 -silesia, level 7, advanced one pass, 4596234 -silesia, level 9, advanced one pass, 4543862 -silesia, level 13, advanced one pass, 4482073 -silesia, level 16, advanced one pass, 4377391 -silesia, level 19, advanced one pass, 4293262 -silesia, no source size, advanced one pass, 4862377 -silesia, long distance mode, advanced one pass, 4853389 -silesia, multithreaded, advanced one pass, 4862377 -silesia, multithreaded long distance mode, advanced one pass, 4853389 -silesia, small window log, advanced one pass, 7126386 -silesia, small hash log, advanced one pass, 6554898 -silesia, small chain log, advanced one pass, 4931093 -silesia, explicit params, advanced one pass, 4815369 -silesia.tar, level -5, advanced one pass, 7160438 -silesia.tar, level -3, advanced one pass, 6789024 -silesia.tar, level -1, advanced one pass, 6195462 -silesia.tar, level 0, advanced one pass, 4875008 -silesia.tar, level 1, advanced one pass, 5339697 -silesia.tar, level 3, advanced one pass, 4875008 -silesia.tar, level 4, advanced one pass, 4813507 -silesia.tar, level 5, advanced one pass, 4722235 -silesia.tar, level 6, advanced one pass, 4672194 -silesia.tar, level 7, advanced one pass, 4606658 -silesia.tar, level 9, advanced one pass, 4554098 -silesia.tar, level 13, advanced one pass, 4491702 -silesia.tar, level 16, advanced one pass, 4381277 -silesia.tar, level 19, advanced one pass, 4281514 -silesia.tar, no source size, advanced one pass, 4875008 -silesia.tar, long distance mode, advanced one pass, 4861218 -silesia.tar, multithreaded, advanced one pass, 4874631 -silesia.tar, multithreaded long distance mode, advanced one pass, 4860683 -silesia.tar, small window log, advanced one pass, 7130394 -silesia.tar, small hash log, advanced one pass, 6587833 -silesia.tar, small chain log, advanced one pass, 4943255 -silesia.tar, explicit params, advanced one pass, 4829974 -github, level -5, advanced one pass, 232744 -github, level -5 with dict, advanced one pass, 46718 -github, level -3, advanced one pass, 220611 -github, level -3 with dict, advanced one pass, 45395 -github, level -1, advanced one pass, 176575 -github, level -1 with dict, advanced one pass, 43170 -github, level 0, advanced one pass, 136397 -github, level 0 with dict, advanced one pass, 41170 -github, level 1, advanced one pass, 143457 -github, level 1 with dict, advanced one pass, 41682 -github, level 3, advanced one pass, 136397 -github, level 3 with dict, advanced one pass, 41170 -github, level 4, advanced one pass, 136144 -github, level 4 with dict, advanced one pass, 41306 -github, level 5, advanced one pass, 135106 -github, level 5 with dict, advanced one pass, 38938 -github, level 6, advanced one pass, 135108 -github, level 6 with dict, advanced one pass, 38632 -github, level 7, advanced one pass, 135108 -github, level 7 with dict, advanced one pass, 38766 -github, level 9, advanced one pass, 135108 -github, level 9 with dict, advanced one pass, 39326 -github, level 13, advanced one pass, 133717 -github, level 13 with dict, advanced one pass, 39716 -github, level 16, advanced one pass, 133717 -github, level 16 with dict, advanced one pass, 37577 -github, level 19, advanced one pass, 133717 -github, level 19 with dict, advanced one pass, 37576 -github, no source size, advanced one pass, 136397 -github, long distance mode, advanced one pass, 136397 -github, multithreaded, advanced one pass, 136397 -github, multithreaded long distance mode, advanced one pass, 136397 -github, small window log, advanced one pass, 136397 -github, small hash log, advanced one pass, 135467 -github, small chain log, advanced one pass, 136314 -github, explicit params, advanced one pass, 137670 -silesia, level -5, advanced one pass small out, 7152294 -silesia, level -3, advanced one pass small out, 6789969 -silesia, level -1, advanced one pass small out, 6191548 -silesia, level 0, advanced one pass small out, 4862377 -silesia, level 1, advanced one pass small out, 5318036 -silesia, level 3, advanced one pass small out, 4862377 -silesia, level 4, advanced one pass small out, 4800629 -silesia, level 5, advanced one pass small out, 4710178 -silesia, level 6, advanced one pass small out, 4659996 -silesia, level 7, advanced one pass small out, 4596234 -silesia, level 9, advanced one pass small out, 4543862 -silesia, level 13, advanced one pass small out, 4482073 -silesia, level 16, advanced one pass small out, 4377391 -silesia, level 19, advanced one pass small out, 4293262 -silesia, no source size, advanced one pass small out, 4862377 -silesia, long distance mode, advanced one pass small out, 4853389 -silesia, multithreaded, advanced one pass small out, 4862377 -silesia, multithreaded long distance mode, advanced one pass small out, 4853389 -silesia, small window log, advanced one pass small out, 7126386 -silesia, small hash log, advanced one pass small out, 6554898 -silesia, small chain log, advanced one pass small out, 4931093 -silesia, explicit params, advanced one pass small out, 4815369 -silesia.tar, level -5, advanced one pass small out, 7160438 -silesia.tar, level -3, advanced one pass small out, 6789024 -silesia.tar, level -1, advanced one pass small out, 6195462 -silesia.tar, level 0, advanced one pass small out, 4875008 -silesia.tar, level 1, advanced one pass small out, 5339697 -silesia.tar, level 3, advanced one pass small out, 4875008 -silesia.tar, level 4, advanced one pass small out, 4813507 -silesia.tar, level 5, advanced one pass small out, 4722235 -silesia.tar, level 6, advanced one pass small out, 4672194 -silesia.tar, level 7, advanced one pass small out, 4606658 -silesia.tar, level 9, advanced one pass small out, 4554098 -silesia.tar, level 13, advanced one pass small out, 4491702 -silesia.tar, level 16, advanced one pass small out, 4381277 -silesia.tar, level 19, advanced one pass small out, 4281514 -silesia.tar, no source size, advanced one pass small out, 4875008 -silesia.tar, long distance mode, advanced one pass small out, 4861218 -silesia.tar, multithreaded, advanced one pass small out, 4874631 -silesia.tar, multithreaded long distance mode, advanced one pass small out, 4860683 -silesia.tar, small window log, advanced one pass small out, 7130394 -silesia.tar, small hash log, advanced one pass small out, 6587833 -silesia.tar, small chain log, advanced one pass small out, 4943255 -silesia.tar, explicit params, advanced one pass small out, 4829974 -github, level -5, advanced one pass small out, 232744 -github, level -5 with dict, advanced one pass small out, 46718 -github, level -3, advanced one pass small out, 220611 -github, level -3 with dict, advanced one pass small out, 45395 -github, level -1, advanced one pass small out, 176575 -github, level -1 with dict, advanced one pass small out, 43170 -github, level 0, advanced one pass small out, 136397 -github, level 0 with dict, advanced one pass small out, 41170 -github, level 1, advanced one pass small out, 143457 -github, level 1 with dict, advanced one pass small out, 41682 -github, level 3, advanced one pass small out, 136397 -github, level 3 with dict, advanced one pass small out, 41170 -github, level 4, advanced one pass small out, 136144 -github, level 4 with dict, advanced one pass small out, 41306 -github, level 5, advanced one pass small out, 135106 -github, level 5 with dict, advanced one pass small out, 38938 -github, level 6, advanced one pass small out, 135108 -github, level 6 with dict, advanced one pass small out, 38632 -github, level 7, advanced one pass small out, 135108 -github, level 7 with dict, advanced one pass small out, 38766 -github, level 9, advanced one pass small out, 135108 -github, level 9 with dict, advanced one pass small out, 39326 -github, level 13, advanced one pass small out, 133717 -github, level 13 with dict, advanced one pass small out, 39716 -github, level 16, advanced one pass small out, 133717 -github, level 16 with dict, advanced one pass small out, 37577 -github, level 19, advanced one pass small out, 133717 -github, level 19 with dict, advanced one pass small out, 37576 -github, no source size, advanced one pass small out, 136397 -github, long distance mode, advanced one pass small out, 136397 -github, multithreaded, advanced one pass small out, 136397 -github, multithreaded long distance mode, advanced one pass small out, 136397 -github, small window log, advanced one pass small out, 136397 -github, small hash log, advanced one pass small out, 135467 -github, small chain log, advanced one pass small out, 136314 -github, explicit params, advanced one pass small out, 137670 -silesia, level -5, advanced streaming, 7152294 -silesia, level -3, advanced streaming, 6789973 -silesia, level -1, advanced streaming, 6191549 -silesia, level 0, advanced streaming, 4862377 -silesia, level 1, advanced streaming, 5318036 -silesia, level 3, advanced streaming, 4862377 -silesia, level 4, advanced streaming, 4800629 -silesia, level 5, advanced streaming, 4710178 -silesia, level 6, advanced streaming, 4659996 -silesia, level 7, advanced streaming, 4596234 -silesia, level 9, advanced streaming, 4543862 -silesia, level 13, advanced streaming, 4482073 -silesia, level 16, advanced streaming, 4377391 -silesia, level 19, advanced streaming, 4293262 -silesia, no source size, advanced streaming, 4862341 -silesia, long distance mode, advanced streaming, 4853389 -silesia, multithreaded, advanced streaming, 4862377 -silesia, multithreaded long distance mode, advanced streaming, 4853389 -silesia, small window log, advanced streaming, 7126389 -silesia, small hash log, advanced streaming, 6554898 -silesia, small chain log, advanced streaming, 4931093 -silesia, explicit params, advanced streaming, 4815380 -silesia.tar, level -5, advanced streaming, 7160440 -silesia.tar, level -3, advanced streaming, 6789026 -silesia.tar, level -1, advanced streaming, 6195465 -silesia.tar, level 0, advanced streaming, 4875010 -silesia.tar, level 1, advanced streaming, 5339701 -silesia.tar, level 3, advanced streaming, 4875010 -silesia.tar, level 4, advanced streaming, 4813507 -silesia.tar, level 5, advanced streaming, 4722240 -silesia.tar, level 6, advanced streaming, 4672203 -silesia.tar, level 7, advanced streaming, 4606658 -silesia.tar, level 9, advanced streaming, 4554105 -silesia.tar, level 13, advanced streaming, 4491703 -silesia.tar, level 16, advanced streaming, 4381277 -silesia.tar, level 19, advanced streaming, 4281514 -silesia.tar, no source size, advanced streaming, 4875006 -silesia.tar, long distance mode, advanced streaming, 4861218 -silesia.tar, multithreaded, advanced streaming, 4875132 -silesia.tar, multithreaded long distance mode, advanced streaming, 4866971 -silesia.tar, small window log, advanced streaming, 7130394 -silesia.tar, small hash log, advanced streaming, 6587834 -silesia.tar, small chain log, advanced streaming, 4943260 -silesia.tar, explicit params, advanced streaming, 4830002 -github, level -5, advanced streaming, 232744 -github, level -5 with dict, advanced streaming, 46718 -github, level -3, advanced streaming, 220611 -github, level -3 with dict, advanced streaming, 45395 -github, level -1, advanced streaming, 176575 -github, level -1 with dict, advanced streaming, 43170 -github, level 0, advanced streaming, 136397 -github, level 0 with dict, advanced streaming, 41170 -github, level 1, advanced streaming, 143457 -github, level 1 with dict, advanced streaming, 41682 -github, level 3, advanced streaming, 136397 -github, level 3 with dict, advanced streaming, 41170 -github, level 4, advanced streaming, 136144 -github, level 4 with dict, advanced streaming, 41306 -github, level 5, advanced streaming, 135106 -github, level 5 with dict, advanced streaming, 38938 -github, level 6, advanced streaming, 135108 -github, level 6 with dict, advanced streaming, 38632 -github, level 7, advanced streaming, 135108 -github, level 7 with dict, advanced streaming, 38766 -github, level 9, advanced streaming, 135108 -github, level 9 with dict, advanced streaming, 39326 -github, level 13, advanced streaming, 133717 -github, level 13 with dict, advanced streaming, 39716 -github, level 16, advanced streaming, 133717 -github, level 16 with dict, advanced streaming, 37577 -github, level 19, advanced streaming, 133717 -github, level 19 with dict, advanced streaming, 37576 -github, no source size, advanced streaming, 136397 -github, long distance mode, advanced streaming, 136397 -github, multithreaded, advanced streaming, 136397 -github, multithreaded long distance mode, advanced streaming, 136397 -github, small window log, advanced streaming, 136397 -github, small hash log, advanced streaming, 135467 -github, small chain log, advanced streaming, 136314 -github, explicit params, advanced streaming, 137670 -silesia, level -5, old streaming, 7152294 -silesia, level -3, old streaming, 6789973 -silesia, level -1, old streaming, 6191549 -silesia, level 0, old streaming, 4862377 -silesia, level 1, old streaming, 5318036 -silesia, level 3, old streaming, 4862377 -silesia, level 4, old streaming, 4800629 -silesia, level 5, old streaming, 4710178 -silesia, level 6, old streaming, 4659996 -silesia, level 7, old streaming, 4596234 -silesia, level 9, old streaming, 4543862 -silesia, level 13, old streaming, 4482073 -silesia, level 16, old streaming, 4377391 -silesia, level 19, old streaming, 4293262 -silesia, no source size, old streaming, 4862341 -silesia.tar, level -5, old streaming, 7160440 -silesia.tar, level -3, old streaming, 6789026 -silesia.tar, level -1, old streaming, 6195465 -silesia.tar, level 0, old streaming, 4875010 -silesia.tar, level 1, old streaming, 5339701 -silesia.tar, level 3, old streaming, 4875010 -silesia.tar, level 4, old streaming, 4813507 -silesia.tar, level 5, old streaming, 4722240 -silesia.tar, level 6, old streaming, 4672203 -silesia.tar, level 7, old streaming, 4606658 -silesia.tar, level 9, old streaming, 4554105 -silesia.tar, level 13, old streaming, 4491703 -silesia.tar, level 16, old streaming, 4381277 -silesia.tar, level 19, old streaming, 4281514 -silesia.tar, no source size, old streaming, 4875006 -github, level -5, old streaming, 232744 -github, level -5 with dict, old streaming, 46718 -github, level -3, old streaming, 220611 -github, level -3 with dict, old streaming, 45395 -github, level -1, old streaming, 176575 -github, level -1 with dict, old streaming, 43170 -github, level 0, old streaming, 136397 -github, level 0 with dict, old streaming, 41170 -github, level 1, old streaming, 143457 -github, level 1 with dict, old streaming, 41682 -github, level 3, old streaming, 136397 -github, level 3 with dict, old streaming, 41170 -github, level 4, old streaming, 136144 -github, level 4 with dict, old streaming, 41306 -github, level 5, old streaming, 135106 -github, level 5 with dict, old streaming, 38938 -github, level 6, old streaming, 135108 -github, level 6 with dict, old streaming, 38632 -github, level 7, old streaming, 135108 -github, level 7 with dict, old streaming, 38766 -github, level 9, old streaming, 135108 -github, level 9 with dict, old streaming, 39326 -github, level 13, old streaming, 133717 -github, level 13 with dict, old streaming, 39716 -github, level 16, old streaming, 133717 -github, level 16 with dict, old streaming, 37577 -github, level 19, old streaming, 133717 -github, level 19 with dict, old streaming, 37576 -github, no source size, old streaming, 141003 +Data, Config, Method, Total compressed size +silesia.tar, level -5, compress simple, 6738558 +silesia.tar, level -3, compress simple, 6446362 +silesia.tar, level -1, compress simple, 6186038 +silesia.tar, level 0, compress simple, 4875008 +silesia.tar, level 1, compress simple, 5334825 +silesia.tar, level 3, compress simple, 4875008 +silesia.tar, level 4, compress simple, 4813507 +silesia.tar, level 5, compress simple, 4722235 +silesia.tar, level 6, compress simple, 4672194 +silesia.tar, level 7, compress simple, 4606658 +silesia.tar, level 9, compress simple, 4554098 +silesia.tar, level 13, compress simple, 4491702 +silesia.tar, level 16, compress simple, 4381277 +silesia.tar, level 19, compress simple, 4281514 +silesia.tar, uncompressed literals, compress simple, 4875008 +silesia.tar, uncompressed literals optimal, compress simple, 4281514 +silesia.tar, huffman literals, compress simple, 6186038 +silesia, level -5, compress cctx, 6737567 +silesia, level -3, compress cctx, 6444663 +silesia, level -1, compress cctx, 6178442 +silesia, level 0, compress cctx, 4862377 +silesia, level 1, compress cctx, 5313144 +silesia, level 3, compress cctx, 4862377 +silesia, level 4, compress cctx, 4800629 +silesia, level 5, compress cctx, 4710178 +silesia, level 6, compress cctx, 4659996 +silesia, level 7, compress cctx, 4596234 +silesia, level 9, compress cctx, 4543862 +silesia, level 13, compress cctx, 4482073 +silesia, level 16, compress cctx, 4377391 +silesia, level 19, compress cctx, 4293262 +silesia, long distance mode, compress cctx, 4862377 +silesia, multithreaded, compress cctx, 4862377 +silesia, multithreaded long distance mode, compress cctx, 4862377 +silesia, small window log, compress cctx, 7115734 +silesia, small hash log, compress cctx, 6554898 +silesia, small chain log, compress cctx, 4931093 +silesia, explicit params, compress cctx, 4813352 +silesia, uncompressed literals, compress cctx, 4862377 +silesia, uncompressed literals optimal, compress cctx, 4293262 +silesia, huffman literals, compress cctx, 6178442 +silesia, multithreaded with advanced params, compress cctx, 4862377 +github, level -5, compress cctx, 205285 +github, level -5 with dict, compress cctx, 47294 +github, level -3, compress cctx, 190643 +github, level -3 with dict, compress cctx, 48047 +github, level -1, compress cctx, 175568 +github, level -1 with dict, compress cctx, 43527 +github, level 0, compress cctx, 136397 +github, level 0 with dict, compress cctx, 41536 +github, level 1, compress cctx, 142450 +github, level 1 with dict, compress cctx, 42157 +github, level 3, compress cctx, 136397 +github, level 3 with dict, compress cctx, 41536 +github, level 4, compress cctx, 136144 +github, level 4 with dict, compress cctx, 41721 +github, level 5, compress cctx, 135106 +github, level 5 with dict, compress cctx, 38934 +github, level 6, compress cctx, 135108 +github, level 6 with dict, compress cctx, 38628 +github, level 7, compress cctx, 135108 +github, level 7 with dict, compress cctx, 38741 +github, level 9, compress cctx, 135108 +github, level 9 with dict, compress cctx, 39335 +github, level 13, compress cctx, 133717 +github, level 13 with dict, compress cctx, 39923 +github, level 16, compress cctx, 133717 +github, level 16 with dict, compress cctx, 37568 +github, level 19, compress cctx, 133717 +github, level 19 with dict, compress cctx, 37567 +github, long distance mode, compress cctx, 141473 +github, multithreaded, compress cctx, 141473 +github, multithreaded long distance mode, compress cctx, 141473 +github, small window log, compress cctx, 141473 +github, small hash log, compress cctx, 138943 +github, small chain log, compress cctx, 139239 +github, explicit params, compress cctx, 140924 +github, uncompressed literals, compress cctx, 136397 +github, uncompressed literals optimal, compress cctx, 133717 +github, huffman literals, compress cctx, 175568 +github, multithreaded with advanced params, compress cctx, 141473 +silesia, level -5, zstdcli, 6882514 +silesia, level -3, zstdcli, 6568406 +silesia, level -1, zstdcli, 6183433 +silesia, level 0, zstdcli, 4862425 +silesia, level 1, zstdcli, 5314157 +silesia, level 3, zstdcli, 4862425 +silesia, level 4, zstdcli, 4800677 +silesia, level 5, zstdcli, 4710226 +silesia, level 6, zstdcli, 4660044 +silesia, level 7, zstdcli, 4596282 +silesia, level 9, zstdcli, 4543910 +silesia, level 13, zstdcli, 4482121 +silesia, level 16, zstdcli, 4377439 +silesia, level 19, zstdcli, 4293310 +silesia, long distance mode, zstdcli, 4853437 +silesia, multithreaded, zstdcli, 4862425 +silesia, multithreaded long distance mode, zstdcli, 4853437 +silesia, small window log, zstdcli, 7126434 +silesia, small hash log, zstdcli, 6554946 +silesia, small chain log, zstdcli, 4931141 +silesia, explicit params, zstdcli, 4815380 +silesia, uncompressed literals, zstdcli, 5155472 +silesia, uncompressed literals optimal, zstdcli, 4325475 +silesia, huffman literals, zstdcli, 5331158 +silesia, multithreaded with advanced params, zstdcli, 5155472 +silesia.tar, level -5, zstdcli, 6738906 +silesia.tar, level -3, zstdcli, 6448409 +silesia.tar, level -1, zstdcli, 6186908 +silesia.tar, level 0, zstdcli, 4875136 +silesia.tar, level 1, zstdcli, 5336255 +silesia.tar, level 3, zstdcli, 4875136 +silesia.tar, level 4, zstdcli, 4814531 +silesia.tar, level 5, zstdcli, 4723284 +silesia.tar, level 6, zstdcli, 4673591 +silesia.tar, level 7, zstdcli, 4608342 +silesia.tar, level 9, zstdcli, 4554700 +silesia.tar, level 13, zstdcli, 4491706 +silesia.tar, level 16, zstdcli, 4381281 +silesia.tar, level 19, zstdcli, 4281518 +silesia.tar, no source size, zstdcli, 4875132 +silesia.tar, long distance mode, zstdcli, 4866975 +silesia.tar, multithreaded, zstdcli, 4875136 +silesia.tar, multithreaded long distance mode, zstdcli, 4866975 +silesia.tar, small window log, zstdcli, 7130434 +silesia.tar, small hash log, zstdcli, 6587841 +silesia.tar, small chain log, zstdcli, 4943259 +silesia.tar, explicit params, zstdcli, 4839202 +silesia.tar, uncompressed literals, zstdcli, 5158134 +silesia.tar, uncompressed literals optimal, zstdcli, 4321098 +silesia.tar, huffman literals, zstdcli, 5347560 +silesia.tar, multithreaded with advanced params, zstdcli, 5158134 +github, level -5, zstdcli, 207285 +github, level -5 with dict, zstdcli, 48718 +github, level -3, zstdcli, 192643 +github, level -3 with dict, zstdcli, 47395 +github, level -1, zstdcli, 177568 +github, level -1 with dict, zstdcli, 45170 +github, level 0, zstdcli, 138397 +github, level 0 with dict, zstdcli, 43170 +github, level 1, zstdcli, 144450 +github, level 1 with dict, zstdcli, 43682 +github, level 3, zstdcli, 138397 +github, level 3 with dict, zstdcli, 43170 +github, level 4, zstdcli, 138144 +github, level 4 with dict, zstdcli, 43306 +github, level 5, zstdcli, 137106 +github, level 5 with dict, zstdcli, 40938 +github, level 6, zstdcli, 137108 +github, level 6 with dict, zstdcli, 40632 +github, level 7, zstdcli, 137108 +github, level 7 with dict, zstdcli, 40766 +github, level 9, zstdcli, 137108 +github, level 9 with dict, zstdcli, 41326 +github, level 13, zstdcli, 135717 +github, level 13 with dict, zstdcli, 41716 +github, level 16, zstdcli, 135717 +github, level 16 with dict, zstdcli, 39577 +github, level 19, zstdcli, 135717 +github, level 19 with dict, zstdcli, 39576 +github, long distance mode, zstdcli, 138397 +github, multithreaded, zstdcli, 138397 +github, multithreaded long distance mode, zstdcli, 138397 +github, small window log, zstdcli, 138397 +github, small hash log, zstdcli, 137467 +github, small chain log, zstdcli, 138314 +github, explicit params, zstdcli, 136140 +github, uncompressed literals, zstdcli, 169004 +github, uncompressed literals optimal, zstdcli, 158824 +github, huffman literals, zstdcli, 144450 +github, multithreaded with advanced params, zstdcli, 169004 +silesia, level -5, advanced one pass, 6737567 +silesia, level -3, advanced one pass, 6444663 +silesia, level -1, advanced one pass, 6178442 +silesia, level 0, advanced one pass, 4862377 +silesia, level 1, advanced one pass, 5313144 +silesia, level 3, advanced one pass, 4862377 +silesia, level 4, advanced one pass, 4800629 +silesia, level 5, advanced one pass, 4710178 +silesia, level 6, advanced one pass, 4659996 +silesia, level 7, advanced one pass, 4596234 +silesia, level 9, advanced one pass, 4543862 +silesia, level 13, advanced one pass, 4482073 +silesia, level 16, advanced one pass, 4377391 +silesia, level 19, advanced one pass, 4293262 +silesia, no source size, advanced one pass, 4862377 +silesia, long distance mode, advanced one pass, 4853389 +silesia, multithreaded, advanced one pass, 4862377 +silesia, multithreaded long distance mode, advanced one pass, 4853389 +silesia, small window log, advanced one pass, 7126386 +silesia, small hash log, advanced one pass, 6554898 +silesia, small chain log, advanced one pass, 4931093 +silesia, explicit params, advanced one pass, 4815369 +silesia, uncompressed literals, advanced one pass, 5155424 +silesia, uncompressed literals optimal, advanced one pass, 4325427 +silesia, huffman literals, advanced one pass, 5326210 +silesia, multithreaded with advanced params, advanced one pass, 5155424 +silesia.tar, level -5, advanced one pass, 6738558 +silesia.tar, level -3, advanced one pass, 6446362 +silesia.tar, level -1, advanced one pass, 6186038 +silesia.tar, level 0, advanced one pass, 4875008 +silesia.tar, level 1, advanced one pass, 5334825 +silesia.tar, level 3, advanced one pass, 4875008 +silesia.tar, level 4, advanced one pass, 4813507 +silesia.tar, level 5, advanced one pass, 4722235 +silesia.tar, level 6, advanced one pass, 4672194 +silesia.tar, level 7, advanced one pass, 4606658 +silesia.tar, level 9, advanced one pass, 4554098 +silesia.tar, level 13, advanced one pass, 4491702 +silesia.tar, level 16, advanced one pass, 4381277 +silesia.tar, level 19, advanced one pass, 4281514 +silesia.tar, no source size, advanced one pass, 4875008 +silesia.tar, long distance mode, advanced one pass, 4861218 +silesia.tar, multithreaded, advanced one pass, 4874631 +silesia.tar, multithreaded long distance mode, advanced one pass, 4860683 +silesia.tar, small window log, advanced one pass, 7130394 +silesia.tar, small hash log, advanced one pass, 6587833 +silesia.tar, small chain log, advanced one pass, 4943255 +silesia.tar, explicit params, advanced one pass, 4829974 +silesia.tar, uncompressed literals, advanced one pass, 5157992 +silesia.tar, uncompressed literals optimal, advanced one pass, 4321094 +silesia.tar, huffman literals, advanced one pass, 5347283 +silesia.tar, multithreaded with advanced params, advanced one pass, 5158545 +github, level -5, advanced one pass, 205285 +github, level -5 with dict, advanced one pass, 46718 +github, level -3, advanced one pass, 190643 +github, level -3 with dict, advanced one pass, 45395 +github, level -1, advanced one pass, 175568 +github, level -1 with dict, advanced one pass, 43170 +github, level 0, advanced one pass, 136397 +github, level 0 with dict, advanced one pass, 41170 +github, level 1, advanced one pass, 142450 +github, level 1 with dict, advanced one pass, 41682 +github, level 3, advanced one pass, 136397 +github, level 3 with dict, advanced one pass, 41170 +github, level 4, advanced one pass, 136144 +github, level 4 with dict, advanced one pass, 41306 +github, level 5, advanced one pass, 135106 +github, level 5 with dict, advanced one pass, 38938 +github, level 6, advanced one pass, 135108 +github, level 6 with dict, advanced one pass, 38632 +github, level 7, advanced one pass, 135108 +github, level 7 with dict, advanced one pass, 38766 +github, level 9, advanced one pass, 135108 +github, level 9 with dict, advanced one pass, 39326 +github, level 13, advanced one pass, 133717 +github, level 13 with dict, advanced one pass, 39716 +github, level 16, advanced one pass, 133717 +github, level 16 with dict, advanced one pass, 37577 +github, level 19, advanced one pass, 133717 +github, level 19 with dict, advanced one pass, 37576 +github, no source size, advanced one pass, 136397 +github, long distance mode, advanced one pass, 136397 +github, multithreaded, advanced one pass, 136397 +github, multithreaded long distance mode, advanced one pass, 136397 +github, small window log, advanced one pass, 136397 +github, small hash log, advanced one pass, 135467 +github, small chain log, advanced one pass, 136314 +github, explicit params, advanced one pass, 137670 +github, uncompressed literals, advanced one pass, 167004 +github, uncompressed literals optimal, advanced one pass, 156824 +github, huffman literals, advanced one pass, 142450 +github, multithreaded with advanced params, advanced one pass, 167004 +silesia, level -5, advanced one pass small out, 6737567 +silesia, level -3, advanced one pass small out, 6444663 +silesia, level -1, advanced one pass small out, 6178442 +silesia, level 0, advanced one pass small out, 4862377 +silesia, level 1, advanced one pass small out, 5313144 +silesia, level 3, advanced one pass small out, 4862377 +silesia, level 4, advanced one pass small out, 4800629 +silesia, level 5, advanced one pass small out, 4710178 +silesia, level 6, advanced one pass small out, 4659996 +silesia, level 7, advanced one pass small out, 4596234 +silesia, level 9, advanced one pass small out, 4543862 +silesia, level 13, advanced one pass small out, 4482073 +silesia, level 16, advanced one pass small out, 4377391 +silesia, level 19, advanced one pass small out, 4293262 +silesia, no source size, advanced one pass small out, 4862377 +silesia, long distance mode, advanced one pass small out, 4853389 +silesia, multithreaded, advanced one pass small out, 4862377 +silesia, multithreaded long distance mode, advanced one pass small out, 4853389 +silesia, small window log, advanced one pass small out, 7126386 +silesia, small hash log, advanced one pass small out, 6554898 +silesia, small chain log, advanced one pass small out, 4931093 +silesia, explicit params, advanced one pass small out, 4815369 +silesia, uncompressed literals, advanced one pass small out, 5155424 +silesia, uncompressed literals optimal, advanced one pass small out, 4325427 +silesia, huffman literals, advanced one pass small out, 5326210 +silesia, multithreaded with advanced params, advanced one pass small out, 5155424 +silesia.tar, level -5, advanced one pass small out, 6738558 +silesia.tar, level -3, advanced one pass small out, 6446362 +silesia.tar, level -1, advanced one pass small out, 6186038 +silesia.tar, level 0, advanced one pass small out, 4875008 +silesia.tar, level 1, advanced one pass small out, 5334825 +silesia.tar, level 3, advanced one pass small out, 4875008 +silesia.tar, level 4, advanced one pass small out, 4813507 +silesia.tar, level 5, advanced one pass small out, 4722235 +silesia.tar, level 6, advanced one pass small out, 4672194 +silesia.tar, level 7, advanced one pass small out, 4606658 +silesia.tar, level 9, advanced one pass small out, 4554098 +silesia.tar, level 13, advanced one pass small out, 4491702 +silesia.tar, level 16, advanced one pass small out, 4381277 +silesia.tar, level 19, advanced one pass small out, 4281514 +silesia.tar, no source size, advanced one pass small out, 4875008 +silesia.tar, long distance mode, advanced one pass small out, 4861218 +silesia.tar, multithreaded, advanced one pass small out, 4874631 +silesia.tar, multithreaded long distance mode, advanced one pass small out, 4860683 +silesia.tar, small window log, advanced one pass small out, 7130394 +silesia.tar, small hash log, advanced one pass small out, 6587833 +silesia.tar, small chain log, advanced one pass small out, 4943255 +silesia.tar, explicit params, advanced one pass small out, 4829974 +silesia.tar, uncompressed literals, advanced one pass small out, 5157992 +silesia.tar, uncompressed literals optimal, advanced one pass small out, 4321094 +silesia.tar, huffman literals, advanced one pass small out, 5347283 +silesia.tar, multithreaded with advanced params, advanced one pass small out, 5158545 +github, level -5, advanced one pass small out, 205285 +github, level -5 with dict, advanced one pass small out, 46718 +github, level -3, advanced one pass small out, 190643 +github, level -3 with dict, advanced one pass small out, 45395 +github, level -1, advanced one pass small out, 175568 +github, level -1 with dict, advanced one pass small out, 43170 +github, level 0, advanced one pass small out, 136397 +github, level 0 with dict, advanced one pass small out, 41170 +github, level 1, advanced one pass small out, 142450 +github, level 1 with dict, advanced one pass small out, 41682 +github, level 3, advanced one pass small out, 136397 +github, level 3 with dict, advanced one pass small out, 41170 +github, level 4, advanced one pass small out, 136144 +github, level 4 with dict, advanced one pass small out, 41306 +github, level 5, advanced one pass small out, 135106 +github, level 5 with dict, advanced one pass small out, 38938 +github, level 6, advanced one pass small out, 135108 +github, level 6 with dict, advanced one pass small out, 38632 +github, level 7, advanced one pass small out, 135108 +github, level 7 with dict, advanced one pass small out, 38766 +github, level 9, advanced one pass small out, 135108 +github, level 9 with dict, advanced one pass small out, 39326 +github, level 13, advanced one pass small out, 133717 +github, level 13 with dict, advanced one pass small out, 39716 +github, level 16, advanced one pass small out, 133717 +github, level 16 with dict, advanced one pass small out, 37577 +github, level 19, advanced one pass small out, 133717 +github, level 19 with dict, advanced one pass small out, 37576 +github, no source size, advanced one pass small out, 136397 +github, long distance mode, advanced one pass small out, 136397 +github, multithreaded, advanced one pass small out, 136397 +github, multithreaded long distance mode, advanced one pass small out, 136397 +github, small window log, advanced one pass small out, 136397 +github, small hash log, advanced one pass small out, 135467 +github, small chain log, advanced one pass small out, 136314 +github, explicit params, advanced one pass small out, 137670 +github, uncompressed literals, advanced one pass small out, 167004 +github, uncompressed literals optimal, advanced one pass small out, 156824 +github, huffman literals, advanced one pass small out, 142450 +github, multithreaded with advanced params, advanced one pass small out, 167004 +silesia, level -5, advanced streaming, 6882466 +silesia, level -3, advanced streaming, 6568358 +silesia, level -1, advanced streaming, 6183385 +silesia, level 0, advanced streaming, 4862377 +silesia, level 1, advanced streaming, 5314109 +silesia, level 3, advanced streaming, 4862377 +silesia, level 4, advanced streaming, 4800629 +silesia, level 5, advanced streaming, 4710178 +silesia, level 6, advanced streaming, 4659996 +silesia, level 7, advanced streaming, 4596234 +silesia, level 9, advanced streaming, 4543862 +silesia, level 13, advanced streaming, 4482073 +silesia, level 16, advanced streaming, 4377391 +silesia, level 19, advanced streaming, 4293262 +silesia, no source size, advanced streaming, 4862341 +silesia, long distance mode, advanced streaming, 4853389 +silesia, multithreaded, advanced streaming, 4862377 +silesia, multithreaded long distance mode, advanced streaming, 4853389 +silesia, small window log, advanced streaming, 7126389 +silesia, small hash log, advanced streaming, 6554898 +silesia, small chain log, advanced streaming, 4931093 +silesia, explicit params, advanced streaming, 4815380 +silesia, uncompressed literals, advanced streaming, 5155424 +silesia, uncompressed literals optimal, advanced streaming, 4325427 +silesia, huffman literals, advanced streaming, 5331110 +silesia, multithreaded with advanced params, advanced streaming, 5155424 +silesia.tar, level -5, advanced streaming, 6982738 +silesia.tar, level -3, advanced streaming, 6641264 +silesia.tar, level -1, advanced streaming, 6190789 +silesia.tar, level 0, advanced streaming, 4875010 +silesia.tar, level 1, advanced streaming, 5336879 +silesia.tar, level 3, advanced streaming, 4875010 +silesia.tar, level 4, advanced streaming, 4813507 +silesia.tar, level 5, advanced streaming, 4722240 +silesia.tar, level 6, advanced streaming, 4672203 +silesia.tar, level 7, advanced streaming, 4606658 +silesia.tar, level 9, advanced streaming, 4554105 +silesia.tar, level 13, advanced streaming, 4491703 +silesia.tar, level 16, advanced streaming, 4381277 +silesia.tar, level 19, advanced streaming, 4281514 +silesia.tar, no source size, advanced streaming, 4875006 +silesia.tar, long distance mode, advanced streaming, 4861218 +silesia.tar, multithreaded, advanced streaming, 4875132 +silesia.tar, multithreaded long distance mode, advanced streaming, 4866971 +silesia.tar, small window log, advanced streaming, 7130394 +silesia.tar, small hash log, advanced streaming, 6587834 +silesia.tar, small chain log, advanced streaming, 4943260 +silesia.tar, explicit params, advanced streaming, 4830002 +silesia.tar, uncompressed literals, advanced streaming, 5157995 +silesia.tar, uncompressed literals optimal, advanced streaming, 4321094 +silesia.tar, huffman literals, advanced streaming, 5352306 +silesia.tar, multithreaded with advanced params, advanced streaming, 5158130 +github, level -5, advanced streaming, 205285 +github, level -5 with dict, advanced streaming, 46718 +github, level -3, advanced streaming, 190643 +github, level -3 with dict, advanced streaming, 45395 +github, level -1, advanced streaming, 175568 +github, level -1 with dict, advanced streaming, 43170 +github, level 0, advanced streaming, 136397 +github, level 0 with dict, advanced streaming, 41170 +github, level 1, advanced streaming, 142450 +github, level 1 with dict, advanced streaming, 41682 +github, level 3, advanced streaming, 136397 +github, level 3 with dict, advanced streaming, 41170 +github, level 4, advanced streaming, 136144 +github, level 4 with dict, advanced streaming, 41306 +github, level 5, advanced streaming, 135106 +github, level 5 with dict, advanced streaming, 38938 +github, level 6, advanced streaming, 135108 +github, level 6 with dict, advanced streaming, 38632 +github, level 7, advanced streaming, 135108 +github, level 7 with dict, advanced streaming, 38766 +github, level 9, advanced streaming, 135108 +github, level 9 with dict, advanced streaming, 39326 +github, level 13, advanced streaming, 133717 +github, level 13 with dict, advanced streaming, 39716 +github, level 16, advanced streaming, 133717 +github, level 16 with dict, advanced streaming, 37577 +github, level 19, advanced streaming, 133717 +github, level 19 with dict, advanced streaming, 37576 +github, no source size, advanced streaming, 136397 +github, long distance mode, advanced streaming, 136397 +github, multithreaded, advanced streaming, 136397 +github, multithreaded long distance mode, advanced streaming, 136397 +github, small window log, advanced streaming, 136397 +github, small hash log, advanced streaming, 135467 +github, small chain log, advanced streaming, 136314 +github, explicit params, advanced streaming, 137670 +github, uncompressed literals, advanced streaming, 167004 +github, uncompressed literals optimal, advanced streaming, 156824 +github, huffman literals, advanced streaming, 142450 +github, multithreaded with advanced params, advanced streaming, 167004 +silesia, level -5, old streaming, 6882466 +silesia, level -3, old streaming, 6568358 +silesia, level -1, old streaming, 6183385 +silesia, level 0, old streaming, 4862377 +silesia, level 1, old streaming, 5314109 +silesia, level 3, old streaming, 4862377 +silesia, level 4, old streaming, 4800629 +silesia, level 5, old streaming, 4710178 +silesia, level 6, old streaming, 4659996 +silesia, level 7, old streaming, 4596234 +silesia, level 9, old streaming, 4543862 +silesia, level 13, old streaming, 4482073 +silesia, level 16, old streaming, 4377391 +silesia, level 19, old streaming, 4293262 +silesia, no source size, old streaming, 4862341 +silesia, long distance mode, old streaming, 12000408 +silesia, multithreaded, old streaming, 12000408 +silesia, multithreaded long distance mode, old streaming, 12000408 +silesia, small window log, old streaming, 12000408 +silesia, small hash log, old streaming, 12000408 +silesia, small chain log, old streaming, 12000408 +silesia, explicit params, old streaming, 12000408 +silesia, uncompressed literals, old streaming, 4862377 +silesia, uncompressed literals optimal, old streaming, 4293262 +silesia, huffman literals, old streaming, 6183385 +silesia, multithreaded with advanced params, old streaming, 12000408 +silesia.tar, level -5, old streaming, 6982738 +silesia.tar, level -3, old streaming, 6641264 +silesia.tar, level -1, old streaming, 6190789 +silesia.tar, level 0, old streaming, 4875010 +silesia.tar, level 1, old streaming, 5336879 +silesia.tar, level 3, old streaming, 4875010 +silesia.tar, level 4, old streaming, 4813507 +silesia.tar, level 5, old streaming, 4722240 +silesia.tar, level 6, old streaming, 4672203 +silesia.tar, level 7, old streaming, 4606658 +silesia.tar, level 9, old streaming, 4554105 +silesia.tar, level 13, old streaming, 4491703 +silesia.tar, level 16, old streaming, 4381277 +silesia.tar, level 19, old streaming, 4281514 +silesia.tar, no source size, old streaming, 4875006 +silesia.tar, long distance mode, old streaming, 12022046 +silesia.tar, multithreaded, old streaming, 12022046 +silesia.tar, multithreaded long distance mode, old streaming, 12022046 +silesia.tar, small window log, old streaming, 12022046 +silesia.tar, small hash log, old streaming, 12022046 +silesia.tar, small chain log, old streaming, 12022046 +silesia.tar, explicit params, old streaming, 12022046 +silesia.tar, uncompressed literals, old streaming, 4875010 +silesia.tar, uncompressed literals optimal, old streaming, 4281514 +silesia.tar, huffman literals, old streaming, 6190789 +silesia.tar, multithreaded with advanced params, old streaming, 12022046 +github, level -5, old streaming, 205285 +github, level -5 with dict, old streaming, 46718 +github, level -3, old streaming, 190643 +github, level -3 with dict, old streaming, 45395 +github, level -1, old streaming, 175568 +github, level -1 with dict, old streaming, 43170 +github, level 0, old streaming, 136397 +github, level 0 with dict, old streaming, 41170 +github, level 1, old streaming, 142450 +github, level 1 with dict, old streaming, 41682 +github, level 3, old streaming, 136397 +github, level 3 with dict, old streaming, 41170 +github, level 4, old streaming, 136144 +github, level 4 with dict, old streaming, 41306 +github, level 5, old streaming, 135106 +github, level 5 with dict, old streaming, 38938 +github, level 6, old streaming, 135108 +github, level 6 with dict, old streaming, 38632 +github, level 7, old streaming, 135108 +github, level 7 with dict, old streaming, 38766 +github, level 9, old streaming, 135108 +github, level 9 with dict, old streaming, 39326 +github, level 13, old streaming, 133717 +github, level 13 with dict, old streaming, 39716 +github, level 16, old streaming, 133717 +github, level 16 with dict, old streaming, 37577 +github, level 19, old streaming, 133717 +github, level 19 with dict, old streaming, 37576 +github, no source size, old streaming, 141003 +github, long distance mode, old streaming, 412933 +github, multithreaded, old streaming, 412933 +github, multithreaded long distance mode, old streaming, 412933 +github, small window log, old streaming, 412933 +github, small hash log, old streaming, 412933 +github, small chain log, old streaming, 412933 +github, explicit params, old streaming, 412933 +github, uncompressed literals, old streaming, 136397 +github, uncompressed literals optimal, old streaming, 133717 +github, huffman literals, old streaming, 175568 +github, multithreaded with advanced params, old streaming, 412933 +silesia, level -5, old streaming advanced, 6882466 +silesia, level -3, old streaming advanced, 6568358 +silesia, level -1, old streaming advanced, 6183385 +silesia, level 0, old streaming advanced, 4862377 +silesia, level 1, old streaming advanced, 5314109 +silesia, level 3, old streaming advanced, 4862377 +silesia, level 4, old streaming advanced, 4800629 +silesia, level 5, old streaming advanced, 4710178 +silesia, level 6, old streaming advanced, 4659996 +silesia, level 7, old streaming advanced, 4596234 +silesia, level 9, old streaming advanced, 4543862 +silesia, level 13, old streaming advanced, 4482073 +silesia, level 16, old streaming advanced, 4377391 +silesia, level 19, old streaming advanced, 4293262 +silesia, no source size, old streaming advanced, 4862341 +silesia, long distance mode, old streaming advanced, 12000408 +silesia, multithreaded, old streaming advanced, 12000408 +silesia, multithreaded long distance mode, old streaming advanced, 12000408 +silesia, small window log, old streaming advanced, 12000408 +silesia, small hash log, old streaming advanced, 12000408 +silesia, small chain log, old streaming advanced, 12000408 +silesia, explicit params, old streaming advanced, 12000408 +silesia, uncompressed literals, old streaming advanced, 4862377 +silesia, uncompressed literals optimal, old streaming advanced, 4293262 +silesia, huffman literals, old streaming advanced, 6183385 +silesia, multithreaded with advanced params, old streaming advanced, 12000408 +silesia.tar, level -5, old streaming advanced, 6982738 +silesia.tar, level -3, old streaming advanced, 6641264 +silesia.tar, level -1, old streaming advanced, 6190789 +silesia.tar, level 0, old streaming advanced, 4875010 +silesia.tar, level 1, old streaming advanced, 5336879 +silesia.tar, level 3, old streaming advanced, 4875010 +silesia.tar, level 4, old streaming advanced, 4813507 +silesia.tar, level 5, old streaming advanced, 4722240 +silesia.tar, level 6, old streaming advanced, 4672203 +silesia.tar, level 7, old streaming advanced, 4606658 +silesia.tar, level 9, old streaming advanced, 4554105 +silesia.tar, level 13, old streaming advanced, 4491703 +silesia.tar, level 16, old streaming advanced, 4381277 +silesia.tar, level 19, old streaming advanced, 4281514 +silesia.tar, no source size, old streaming advanced, 4875006 +silesia.tar, long distance mode, old streaming advanced, 12022046 +silesia.tar, multithreaded, old streaming advanced, 12022046 +silesia.tar, multithreaded long distance mode, old streaming advanced, 12022046 +silesia.tar, small window log, old streaming advanced, 12022046 +silesia.tar, small hash log, old streaming advanced, 12022046 +silesia.tar, small chain log, old streaming advanced, 12022046 +silesia.tar, explicit params, old streaming advanced, 12022046 +silesia.tar, uncompressed literals, old streaming advanced, 4875010 +silesia.tar, uncompressed literals optimal, old streaming advanced, 4281514 +silesia.tar, huffman literals, old streaming advanced, 6190789 +silesia.tar, multithreaded with advanced params, old streaming advanced, 12022046 +github, level -5, old streaming advanced, 205285 +github, level -5 with dict, old streaming advanced, 46718 +github, level -3, old streaming advanced, 190643 +github, level -3 with dict, old streaming advanced, 45395 +github, level -1, old streaming advanced, 175568 +github, level -1 with dict, old streaming advanced, 43170 +github, level 0, old streaming advanced, 136397 +github, level 0 with dict, old streaming advanced, 41170 +github, level 1, old streaming advanced, 142450 +github, level 1 with dict, old streaming advanced, 41682 +github, level 3, old streaming advanced, 136397 +github, level 3 with dict, old streaming advanced, 41170 +github, level 4, old streaming advanced, 136144 +github, level 4 with dict, old streaming advanced, 41306 +github, level 5, old streaming advanced, 135106 +github, level 5 with dict, old streaming advanced, 38938 +github, level 6, old streaming advanced, 135108 +github, level 6 with dict, old streaming advanced, 38632 +github, level 7, old streaming advanced, 135108 +github, level 7 with dict, old streaming advanced, 38766 +github, level 9, old streaming advanced, 135108 +github, level 9 with dict, old streaming advanced, 39326 +github, level 13, old streaming advanced, 133717 +github, level 13 with dict, old streaming advanced, 39716 +github, level 16, old streaming advanced, 133717 +github, level 16 with dict, old streaming advanced, 37577 +github, level 19, old streaming advanced, 133717 +github, level 19 with dict, old streaming advanced, 37576 +github, no source size, old streaming advanced, 141003 +github, long distance mode, old streaming advanced, 412933 +github, multithreaded, old streaming advanced, 412933 +github, multithreaded long distance mode, old streaming advanced, 412933 +github, small window log, old streaming advanced, 412933 +github, small hash log, old streaming advanced, 412933 +github, small chain log, old streaming advanced, 412933 +github, explicit params, old streaming advanced, 412933 +github, uncompressed literals, old streaming advanced, 136397 +github, uncompressed literals optimal, old streaming advanced, 133717 +github, huffman literals, old streaming advanced, 175568 +github, multithreaded with advanced params, old streaming advanced, 412933 +silesia, level -5, old streaming cdcit, 6882466 +silesia, level -3, old streaming cdcit, 6568358 +silesia, level -1, old streaming cdcit, 6183385 +silesia, level 0, old streaming cdcit, 4862377 +silesia, level 1, old streaming cdcit, 5314109 +silesia, level 3, old streaming cdcit, 4862377 +silesia, level 4, old streaming cdcit, 4800629 +silesia, level 5, old streaming cdcit, 4710178 +silesia, level 6, old streaming cdcit, 4659996 +silesia, level 7, old streaming cdcit, 4596234 +silesia, level 9, old streaming cdcit, 4543862 +silesia, level 13, old streaming cdcit, 4482073 +silesia, level 16, old streaming cdcit, 4377391 +silesia, level 19, old streaming cdcit, 4293262 +silesia, no source size, old streaming cdcit, 4862341 +silesia, long distance mode, old streaming cdcit, 12000408 +silesia, multithreaded, old streaming cdcit, 12000408 +silesia, multithreaded long distance mode, old streaming cdcit, 12000408 +silesia, small window log, old streaming cdcit, 12000408 +silesia, small hash log, old streaming cdcit, 12000408 +silesia, small chain log, old streaming cdcit, 12000408 +silesia, explicit params, old streaming cdcit, 12000408 +silesia, uncompressed literals, old streaming cdcit, 4862377 +silesia, uncompressed literals optimal, old streaming cdcit, 4293262 +silesia, huffman literals, old streaming cdcit, 6183385 +silesia, multithreaded with advanced params, old streaming cdcit, 12000408 +silesia.tar, level -5, old streaming cdcit, 6982738 +silesia.tar, level -3, old streaming cdcit, 6641264 +silesia.tar, level -1, old streaming cdcit, 6190789 +silesia.tar, level 0, old streaming cdcit, 4875010 +silesia.tar, level 1, old streaming cdcit, 5336879 +silesia.tar, level 3, old streaming cdcit, 4875010 +silesia.tar, level 4, old streaming cdcit, 4813507 +silesia.tar, level 5, old streaming cdcit, 4722240 +silesia.tar, level 6, old streaming cdcit, 4672203 +silesia.tar, level 7, old streaming cdcit, 4606658 +silesia.tar, level 9, old streaming cdcit, 4554105 +silesia.tar, level 13, old streaming cdcit, 4491703 +silesia.tar, level 16, old streaming cdcit, 4381277 +silesia.tar, level 19, old streaming cdcit, 4281514 +silesia.tar, no source size, old streaming cdcit, 4875006 +silesia.tar, long distance mode, old streaming cdcit, 12022046 +silesia.tar, multithreaded, old streaming cdcit, 12022046 +silesia.tar, multithreaded long distance mode, old streaming cdcit, 12022046 +silesia.tar, small window log, old streaming cdcit, 12022046 +silesia.tar, small hash log, old streaming cdcit, 12022046 +silesia.tar, small chain log, old streaming cdcit, 12022046 +silesia.tar, explicit params, old streaming cdcit, 12022046 +silesia.tar, uncompressed literals, old streaming cdcit, 4875010 +silesia.tar, uncompressed literals optimal, old streaming cdcit, 4281514 +silesia.tar, huffman literals, old streaming cdcit, 6190789 +silesia.tar, multithreaded with advanced params, old streaming cdcit, 12022046 +github, level -5, old streaming cdcit, 205285 +github, level -5 with dict, old streaming cdcit, 46718 +github, level -3, old streaming cdcit, 190643 +github, level -3 with dict, old streaming cdcit, 45395 +github, level -1, old streaming cdcit, 175568 +github, level -1 with dict, old streaming cdcit, 43170 +github, level 0, old streaming cdcit, 136397 +github, level 0 with dict, old streaming cdcit, 41170 +github, level 1, old streaming cdcit, 142450 +github, level 1 with dict, old streaming cdcit, 41682 +github, level 3, old streaming cdcit, 136397 +github, level 3 with dict, old streaming cdcit, 41170 +github, level 4, old streaming cdcit, 136144 +github, level 4 with dict, old streaming cdcit, 41306 +github, level 5, old streaming cdcit, 135106 +github, level 5 with dict, old streaming cdcit, 38938 +github, level 6, old streaming cdcit, 135108 +github, level 6 with dict, old streaming cdcit, 38632 +github, level 7, old streaming cdcit, 135108 +github, level 7 with dict, old streaming cdcit, 38766 +github, level 9, old streaming cdcit, 135108 +github, level 9 with dict, old streaming cdcit, 39326 +github, level 13, old streaming cdcit, 133717 +github, level 13 with dict, old streaming cdcit, 39716 +github, level 16, old streaming cdcit, 133717 +github, level 16 with dict, old streaming cdcit, 37577 +github, level 19, old streaming cdcit, 133717 +github, level 19 with dict, old streaming cdcit, 37576 +github, no source size, old streaming cdcit, 141003 +github, long distance mode, old streaming cdcit, 412933 +github, multithreaded, old streaming cdcit, 412933 +github, multithreaded long distance mode, old streaming cdcit, 412933 +github, small window log, old streaming cdcit, 412933 +github, small hash log, old streaming cdcit, 412933 +github, small chain log, old streaming cdcit, 412933 +github, explicit params, old streaming cdcit, 412933 +github, uncompressed literals, old streaming cdcit, 136397 +github, uncompressed literals optimal, old streaming cdcit, 133717 +github, huffman literals, old streaming cdcit, 175568 +github, multithreaded with advanced params, old streaming cdcit, 412933 +silesia, level -5, old streaming advanced cdict, 6882466 +silesia, level -3, old streaming advanced cdict, 6568358 +silesia, level -1, old streaming advanced cdict, 6183385 +silesia, level 0, old streaming advanced cdict, 4862377 +silesia, level 1, old streaming advanced cdict, 5314109 +silesia, level 3, old streaming advanced cdict, 4862377 +silesia, level 4, old streaming advanced cdict, 4800629 +silesia, level 5, old streaming advanced cdict, 4710178 +silesia, level 6, old streaming advanced cdict, 4659996 +silesia, level 7, old streaming advanced cdict, 4596234 +silesia, level 9, old streaming advanced cdict, 4543862 +silesia, level 13, old streaming advanced cdict, 4482073 +silesia, level 16, old streaming advanced cdict, 4377391 +silesia, level 19, old streaming advanced cdict, 4293262 +silesia, no source size, old streaming advanced cdict, 4862341 +silesia, long distance mode, old streaming advanced cdict, 12000408 +silesia, multithreaded, old streaming advanced cdict, 12000408 +silesia, multithreaded long distance mode, old streaming advanced cdict, 12000408 +silesia, small window log, old streaming advanced cdict, 12000408 +silesia, small hash log, old streaming advanced cdict, 12000408 +silesia, small chain log, old streaming advanced cdict, 12000408 +silesia, explicit params, old streaming advanced cdict, 12000408 +silesia, uncompressed literals, old streaming advanced cdict, 4862377 +silesia, uncompressed literals optimal, old streaming advanced cdict, 4293262 +silesia, huffman literals, old streaming advanced cdict, 6183385 +silesia, multithreaded with advanced params, old streaming advanced cdict, 12000408 +silesia.tar, level -5, old streaming advanced cdict, 6982738 +silesia.tar, level -3, old streaming advanced cdict, 6641264 +silesia.tar, level -1, old streaming advanced cdict, 6190789 +silesia.tar, level 0, old streaming advanced cdict, 4875010 +silesia.tar, level 1, old streaming advanced cdict, 5336879 +silesia.tar, level 3, old streaming advanced cdict, 4875010 +silesia.tar, level 4, old streaming advanced cdict, 4813507 +silesia.tar, level 5, old streaming advanced cdict, 4722240 +silesia.tar, level 6, old streaming advanced cdict, 4672203 +silesia.tar, level 7, old streaming advanced cdict, 4606658 +silesia.tar, level 9, old streaming advanced cdict, 4554105 +silesia.tar, level 13, old streaming advanced cdict, 4491703 +silesia.tar, level 16, old streaming advanced cdict, 4381277 +silesia.tar, level 19, old streaming advanced cdict, 4281514 +silesia.tar, no source size, old streaming advanced cdict, 4875006 +silesia.tar, long distance mode, old streaming advanced cdict, 12022046 +silesia.tar, multithreaded, old streaming advanced cdict, 12022046 +silesia.tar, multithreaded long distance mode, old streaming advanced cdict, 12022046 +silesia.tar, small window log, old streaming advanced cdict, 12022046 +silesia.tar, small hash log, old streaming advanced cdict, 12022046 +silesia.tar, small chain log, old streaming advanced cdict, 12022046 +silesia.tar, explicit params, old streaming advanced cdict, 12022046 +silesia.tar, uncompressed literals, old streaming advanced cdict, 4875010 +silesia.tar, uncompressed literals optimal, old streaming advanced cdict, 4281514 +silesia.tar, huffman literals, old streaming advanced cdict, 6190789 +silesia.tar, multithreaded with advanced params, old streaming advanced cdict, 12022046 +github, level -5, old streaming advanced cdict, 205285 +github, level -5 with dict, old streaming advanced cdict, 46718 +github, level -3, old streaming advanced cdict, 190643 +github, level -3 with dict, old streaming advanced cdict, 45395 +github, level -1, old streaming advanced cdict, 175568 +github, level -1 with dict, old streaming advanced cdict, 43170 +github, level 0, old streaming advanced cdict, 136397 +github, level 0 with dict, old streaming advanced cdict, 41170 +github, level 1, old streaming advanced cdict, 142450 +github, level 1 with dict, old streaming advanced cdict, 41682 +github, level 3, old streaming advanced cdict, 136397 +github, level 3 with dict, old streaming advanced cdict, 41170 +github, level 4, old streaming advanced cdict, 136144 +github, level 4 with dict, old streaming advanced cdict, 41306 +github, level 5, old streaming advanced cdict, 135106 +github, level 5 with dict, old streaming advanced cdict, 38938 +github, level 6, old streaming advanced cdict, 135108 +github, level 6 with dict, old streaming advanced cdict, 38632 +github, level 7, old streaming advanced cdict, 135108 +github, level 7 with dict, old streaming advanced cdict, 38766 +github, level 9, old streaming advanced cdict, 135108 +github, level 9 with dict, old streaming advanced cdict, 39326 +github, level 13, old streaming advanced cdict, 133717 +github, level 13 with dict, old streaming advanced cdict, 39716 +github, level 16, old streaming advanced cdict, 133717 +github, level 16 with dict, old streaming advanced cdict, 37577 +github, level 19, old streaming advanced cdict, 133717 +github, level 19 with dict, old streaming advanced cdict, 37576 +github, no source size, old streaming advanced cdict, 141003 +github, long distance mode, old streaming advanced cdict, 412933 +github, multithreaded, old streaming advanced cdict, 412933 +github, multithreaded long distance mode, old streaming advanced cdict, 412933 +github, small window log, old streaming advanced cdict, 412933 +github, small hash log, old streaming advanced cdict, 412933 +github, small chain log, old streaming advanced cdict, 412933 +github, explicit params, old streaming advanced cdict, 412933 +github, uncompressed literals, old streaming advanced cdict, 136397 +github, uncompressed literals optimal, old streaming advanced cdict, 133717 +github, huffman literals, old streaming advanced cdict, 175568 +github, multithreaded with advanced params, old streaming advanced cdict, 412933 diff --git a/tests/regression/test.c b/tests/regression/test.c index 9e7b83c19daf..812893b2730e 100644 --- a/tests/regression/test.c +++ b/tests/regression/test.c @@ -153,7 +153,7 @@ static void print_help(void) { } } -/** Parse the arguments. Teturn 0 on success. Print help on failure. */ +/** Parse the arguments. Return 0 on success. Print help on failure. */ static int parse_args(int argc, char** argv) { int option_index = 0; int c; diff --git a/tests/roundTripCrash.c b/tests/roundTripCrash.c index 3f4ace8c9800..3de5933185d5 100644 --- a/tests/roundTripCrash.c +++ b/tests/roundTripCrash.c @@ -93,9 +93,9 @@ static size_t cctxParamRoundTripTest(void* resultBuff, size_t resultBuffCapacity int const cLevel = h32 % maxClevel; /* Set parameters */ - CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_c_compressionLevel, cLevel) ); - CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_c_nbWorkers, 2) ); - CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_c_overlapLog, 5) ); + CHECK_Z( ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_compressionLevel, cLevel) ); + CHECK_Z( ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_nbWorkers, 2) ); + CHECK_Z( ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_overlapLog, 5) ); /* Apply parameters */ diff --git a/tests/symbols.c b/tests/symbols.c index 600d81670a55..4d9c6fc0c9e2 100644 --- a/tests/symbols.c +++ b/tests/symbols.c @@ -31,6 +31,7 @@ static const void *symbols[] = { &ZSTD_getFrameContentSize, &ZSTD_maxCLevel, &ZSTD_compressBound, + &ZSTD_decompressBound, &ZSTD_isError, &ZSTD_getErrorName, &ZSTD_createCCtx, diff --git a/tests/zbufftest.c b/tests/zbufftest.c index 9b6f7bad6dc6..8cbde3f4f38e 100644 --- a/tests/zbufftest.c +++ b/tests/zbufftest.c @@ -25,6 +25,7 @@ #include <stdlib.h> /* free */ #include <stdio.h> /* fgets, sscanf */ #include <string.h> /* strcmp */ +#include "timefn.h" /* UTIL_time_t */ #include "mem.h" #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */ #include "zstd.h" /* ZSTD_compressBound */ diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index ac300890415f..55c14ad5995f 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -26,6 +26,7 @@ #include <stdio.h> /* fgets, sscanf */ #include <string.h> /* strcmp */ #include <assert.h> /* assert */ +#include "timefn.h" /* UTIL_time_t, UTIL_getTime */ #include "mem.h" #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel, ZSTD_customMem, ZSTD_getDictID_fromFrame */ #include "zstd.h" /* ZSTD_compressBound */ @@ -37,6 +38,7 @@ #include "xxhash.h" /* XXH64_* */ #include "seqgen.h" #include "util.h" +#include "timefn.h" /* UTIL_time_t, UTIL_clockSpanMicro, UTIL_getTime */ /*-************************************ @@ -344,6 +346,20 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)(cstreamSize + cdictSize)); } + /* context size functions */ + DISPLAYLEVEL(3, "test%3i : estimate CStream size using CCtxParams : ", testNb++); + { ZSTD_CCtx_params* const params = ZSTD_createCCtxParams(); + size_t cstreamSize, cctxSize; + CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_compressionLevel, 19) ); + cstreamSize = ZSTD_estimateCStreamSize_usingCCtxParams(params); + CHECK_Z(cstreamSize); + cctxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params); + CHECK_Z(cctxSize); + if (cstreamSize <= cctxSize + 2 * ZSTD_BLOCKSIZE_MAX) goto _output_error; + ZSTD_freeCCtxParams(params); + DISPLAYLEVEL(3, "OK \n"); + } + DISPLAYLEVEL(3, "test%3i : check actual CStream size : ", testNb++); { size_t const s = ZSTD_sizeof_CStream(zc); if (ZSTD_isError(s)) goto _output_error; @@ -495,7 +511,7 @@ static int basicUnitTests(U32 seed, double compressibility) /* _srcSize compression test */ DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); - ZSTD_initCStream_srcSize(zc, 1, CNBufferSize); + CHECK_Z( ZSTD_initCStream_srcSize(zc, 1, CNBufferSize) ); outBuff.dst = (char*)(compressedBuffer); outBuff.size = compressedBufferSize; outBuff.pos = 0; @@ -503,11 +519,14 @@ static int basicUnitTests(U32 seed, double compressibility) inBuff.size = CNBufferSize; inBuff.pos = 0; CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); - if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ - { size_t const r = ZSTD_endStream(zc, &outBuff); - if (r != 0) goto _output_error; } /* error, or some data not flushed */ - { unsigned long long origSize = ZSTD_findDecompressedSize(outBuff.dst, outBuff.pos); - if ((size_t)origSize != CNBufferSize) goto _output_error; } /* exact original size must be present */ + CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed"); + { size_t const r = ZSTD_endStream(zc, &outBuff); + CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r); + } + { unsigned long long origSize = ZSTD_findDecompressedSize(outBuff.dst, outBuff.pos); + CHECK(origSize == ZSTD_CONTENTSIZE_UNKNOWN, "Unknown!"); + CHECK((size_t)origSize != CNBufferSize, "Exact original size must be present (got %llu)", origSize); + } DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); /* wrong _srcSize compression test */ @@ -706,6 +725,156 @@ static int basicUnitTests(U32 seed, double compressibility) } DISPLAYLEVEL(3, "OK\n"); + ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters); + CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dictionary.start, dictionary.filled) ); + cSize = ZSTD_compress2(zc, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBufferSize, 100 KB)); + CHECK_Z(cSize); + DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() with dictionary : ", testNb++); + { + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + /* We should fail to decompress without a dictionary. */ + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); + { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0}; + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + size_t const ret = ZSTD_decompressStream(dctx, &out, &in); + if (!ZSTD_isError(ret)) goto _output_error; + } + /* We should succeed to decompress with the dictionary. */ + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); + CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictionary.start, dictionary.filled) ); + { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0}; + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error; + if (in.pos != in.size) goto _output_error; + } + /* The dictionary should presist across calls. */ + { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0}; + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error; + if (in.pos != in.size) goto _output_error; + } + /* The dictionary should not be cleared by ZSTD_reset_session_only. */ + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only); + { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0}; + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error; + if (in.pos != in.size) goto _output_error; + } + /* When we reset the context the dictionary is cleared. */ + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); + { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0}; + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + size_t const ret = ZSTD_decompressStream(dctx, &out, &in); + if (!ZSTD_isError(ret)) goto _output_error; + } + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_resetDStream() with dictionary : ", testNb++); + { + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + /* We should succeed to decompress with the dictionary. */ + ZSTD_resetDStream(dctx); + CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictionary.start, dictionary.filled) ); + { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0}; + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error; + if (in.pos != in.size) goto _output_error; + } + /* The dictionary should not be cleared by ZSTD_resetDStream(). */ + ZSTD_resetDStream(dctx); + { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0}; + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error; + if (in.pos != in.size) goto _output_error; + } + /* The dictionary should be cleared by ZSTD_initDStream(). */ + CHECK_Z( ZSTD_initDStream(dctx) ); + { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0}; + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + size_t const ret = ZSTD_decompressStream(dctx, &out, &in); + if (!ZSTD_isError(ret)) goto _output_error; + } + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() with ddict : ", testNb++); + { + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + ZSTD_DDict* ddict = ZSTD_createDDict(dictionary.start, dictionary.filled); + /* We should succeed to decompress with the ddict. */ + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); + CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) ); + { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0}; + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error; + if (in.pos != in.size) goto _output_error; + } + /* The ddict should presist across calls. */ + { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0}; + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error; + if (in.pos != in.size) goto _output_error; + } + /* When we reset the context the ddict is cleared. */ + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); + { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0}; + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + size_t const ret = ZSTD_decompressStream(dctx, &out, &in); + if (!ZSTD_isError(ret)) goto _output_error; + } + ZSTD_freeDCtx(dctx); + ZSTD_freeDDict(ddict); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++); + { + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + /* We should succeed to decompress with the prefix. */ + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); + CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictionary.start, dictionary.filled, ZSTD_dct_auto) ); + { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0}; + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error; + if (in.pos != in.size) goto _output_error; + } + /* The prefix should be cleared after the first compression. */ + { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0}; + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + size_t const ret = ZSTD_decompressStream(dctx, &out, &in); + if (!ZSTD_isError(ret)) goto _output_error; + } + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_initDStream*() with dictionary : ", testNb++); + { + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + ZSTD_DDict* ddict = ZSTD_createDDict(dictionary.start, dictionary.filled); + size_t ret; + /* We should succeed to decompress with the dictionary. */ + CHECK_Z( ZSTD_initDStream_usingDict(dctx, dictionary.start, dictionary.filled) ); + CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) ); + /* The dictionary should presist across calls. */ + CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) ); + /* We should succeed to decompress with the ddict. */ + CHECK_Z( ZSTD_initDStream_usingDDict(dctx, ddict) ); + CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) ); + /* The ddict should presist across calls. */ + CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) ); + /* When we reset the context the ddict is cleared. */ + CHECK_Z( ZSTD_initDStream(dctx) ); + ret = ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize); + if (!ZSTD_isError(ret)) goto _output_error; + ZSTD_freeDCtx(dctx); + ZSTD_freeDDict(ddict); + } + DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_usingCDict_advanced with masked dictID : ", testNb++); { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictionary.filled); ZSTD_frameParameters const fParams = { 1 /* contentSize */, 1 /* checksum */, 1 /* noDictID */}; @@ -882,7 +1051,7 @@ static int basicUnitTests(U32 seed, double compressibility) inBuff.size = srcSize; assert(srcSize < COMPRESSIBLE_NOISE_LENGTH); inBuff.pos = 0; } - { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 4 KB, dictionary.filled); /* intentionnally lies on estimatedSrcSize, to push cdict into targeting a small window size */ + { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 4 KB, dictionary.filled); /* intentionally lies on estimatedSrcSize, to push cdict into targeting a small window size */ ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem); DISPLAYLEVEL(5, "cParams.windowLog = %u : ", cParams.windowLog); CHECK_Z( ZSTD_CCtx_refCDict(zc, cdict) ); @@ -1703,7 +1872,7 @@ static size_t setCCtxParameter(ZSTD_CCtx* zc, ZSTD_CCtx_params* cctxParams, int useOpaqueAPI) { if (useOpaqueAPI) { - return ZSTD_CCtxParam_setParameter(cctxParams, param, value); + return ZSTD_CCtxParams_setParameter(cctxParams, param, value); } else { return ZSTD_CCtx_setParameter(zc, param, value); } @@ -1900,7 +2069,7 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest, CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) ); } - /* multi-threading parameters. Only adjust ocassionally for small tests. */ + /* multi-threading parameters. Only adjust occasionally for small tests. */ if (bigTests || (FUZ_rand(&lseed) & 0xF) == 0xF) { U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1; U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1; @@ -1930,16 +2099,6 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest, } else { CHECK_Z( ZSTD_CCtx_loadDictionary_byReference(zc, dict, dictSize) ); } - if (dict && dictSize) { - /* test that compression parameters are rejected (correctly) after loading a non-NULL dictionary */ - if (opaqueAPI) { - size_t const setError = ZSTD_CCtx_setParametersUsingCCtxParams(zc, cctxParams); - CHECK(!ZSTD_isError(setError), "ZSTD_CCtx_setParametersUsingCCtxParams should have failed"); - } else { - size_t const setError = ZSTD_CCtx_setParameter(zc, ZSTD_c_windowLog, cParams.windowLog-1); - CHECK(!ZSTD_isError(setError), "ZSTD_CCtx_setParameter should have failed"); - } - } } else { CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) ); } |