diff options
Diffstat (limited to 'tests')
31 files changed, 4062 insertions, 817 deletions
diff --git a/tests/Makefile b/tests/Makefile index 2a96829f61c5..25bd5c84ef90 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -132,18 +132,18 @@ fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP) fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD) fullbench fullbench32 : DEBUGFLAGS = -DNDEBUG # turn off assert() for speed measurements fullbench fullbench32 : $(ZSTD_FILES) -fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/bench.c fullbench.c +fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c fullbench.c $(CC) $(FLAGS) $^ -o $@$(EXT) fullbench-lib : CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ fullbench-lib : zstd-staticLib -fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/bench.c fullbench.c +fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c fullbench.c $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) $(ZSTDDIR)/libzstd.a # note : broken : requires unavailable symbols fullbench-dll : zstd-dll fullbench-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd -fullbench-dll: $(PRGDIR)/datagen.c fullbench.c +fullbench-dll: $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c fullbench.c # $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) @@ -152,32 +152,32 @@ fuzzer : LDFLAGS += $(MULTITHREAD_LD) fuzzer32: CFLAGS += -m32 fuzzer : $(ZSTDMT_OBJECTS) fuzzer32: $(ZSTD_FILES) -fuzzer fuzzer32 : $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c +fuzzer fuzzer32 : $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/datagen.c fuzzer.c $(CC) $(FLAGS) $^ -o $@$(EXT) fuzzer-dll : zstd-dll fuzzer-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd -fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c fuzzer.c +fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/datagen.c fuzzer.c $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) zbufftest : CPPFLAGS += -I$(ZSTDDIR)/deprecated zbufftest : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings -zbufftest : $(ZSTD_OBJECTS) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c +zbufftest : $(ZSTD_OBJECTS) $(ZBUFF_FILES) $(PRGDIR)/util.c $(PRGDIR)/datagen.c zbufftest.c $(CC) $(FLAGS) $^ -o $@$(EXT) zbufftest32 : CPPFLAGS += -I$(ZSTDDIR)/deprecated zbufftest32 : CFLAGS += -Wno-deprecated-declarations -m32 -zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c +zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/util.c $(PRGDIR)/datagen.c zbufftest.c $(CC) $(FLAGS) $^ -o $@$(EXT) zbufftest-dll : zstd-dll zbufftest-dll : CPPFLAGS += -I$(ZSTDDIR)/deprecated zbufftest-dll : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings zbufftest-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd -zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zbufftest.c +zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/datagen.c zbufftest.c $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) -ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c seqgen.c zstreamtest.c +ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c $(PRGDIR)/util.c seqgen.c zstreamtest.c ZSTREAM_PROPER_FILES := $(ZDICT_FILES) $(ZSTREAM_LOCAL_FILES) ZSTREAMFILES := $(ZSTD_FILES) $(ZSTREAM_PROPER_FILES) zstreamtest32 : CFLAGS += -m32 @@ -203,7 +203,7 @@ zstreamtest-dll : $(ZSTREAM_LOCAL_FILES) $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) paramgrill : DEBUGFLAGS = # turn off assert() by default for speed measurements -paramgrill : $(ZSTD_FILES) $(PRGDIR)/bench.c $(PRGDIR)/datagen.c paramgrill.c +paramgrill : $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/benchfn.c $(PRGDIR)/benchzstd.c $(PRGDIR)/datagen.c paramgrill.c $(CC) $(FLAGS) $^ -lm -o $@$(EXT) datagen : $(PRGDIR)/datagen.c datagencli.c @@ -222,7 +222,7 @@ legacy : CPPFLAGS += -I$(ZSTDDIR)/legacy -DZSTD_LEGACY_SUPPORT=4 legacy : $(ZSTD_FILES) $(wildcard $(ZSTDDIR)/legacy/*.c) legacy.c $(CC) $(FLAGS) $^ -o $@$(EXT) -decodecorpus : $(filter-out zstdc_zstd_compress.o, $(ZSTD_OBJECTS)) $(ZDICT_FILES) decodecorpus.c +decodecorpus : $(filter-out zstdc_zstd_compress.o, $(ZSTD_OBJECTS)) $(ZDICT_FILES) $(PRGDIR)/util.c decodecorpus.c $(CC) $(FLAGS) $^ -o $@$(EXT) -lm symbols : symbols.c zstd-dll @@ -233,7 +233,7 @@ else $(CC) $(FLAGS) $< -o $@$(EXT) -Wl,-rpath=$(ZSTDDIR) $(ZSTDDIR)/libzstd.so # broken on Mac endif -poolTests : poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/error_private.c +poolTests : $(PRGDIR)/util.c poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/error_private.c $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT) .PHONY: versionsTest @@ -320,7 +320,7 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32 test-all: test test32 valgrindTest test-decodecorpus-cli -.PHONY: test-zstd test-zstd32 test-zstd-nolegacy +.PHONY: test-zstd test-zstd32 test-zstd-nolegacy test-zstdgrep test-zstd: ZSTD = $(PRGDIR)/zstd test-zstd: zstd @@ -352,6 +352,10 @@ test-gzstd: gzstd $(PRGDIR)/zstd -dcf - <hello_zst_gz_txt.gz $(RM) *.gz *.zst README2.md gz_zstd zstd_gz hello.txt +test-zstdgrep: gzstd + @echo a | $(PRGDIR)/zstd | $(PRGDIR)/zstdgrep a + @echo a | $(PRGDIR)/zstd | $(PRGDIR)/zstdgrep b && return 1 || return 0 + test-fullbench: fullbench datagen $(QEMU_SYS) ./fullbench -i1 $(QEMU_SYS) ./fullbench -i1 -P0 diff --git a/tests/README.md b/tests/README.md index f28766bd1942..7c6fb0db845c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -41,7 +41,7 @@ Additional remarks: The example usage with two test files, one e-mail address, and with an additional message: ``` ./test-zstd-speed.py "silesia.tar calgary.tar" "email@gmail.com" --message "tested on my laptop" --sleepTime 60 -``` +``` To run the script in background please use: ``` @@ -100,19 +100,19 @@ Full list of arguments h# - hashLog c# - chainLog s# - searchLog - l# - searchLength + l# - minMatch t# - targetLength S# - strategy L# - level --zstd= : Single run, parameter selection syntax same as zstdcli with more parameters - (Added forceAttachDictionary / fadt) - When invoked with --optimize, this represents the sample to exceed. + (Added forceAttachDictionary / fadt) + When invoked with --optimize, this represents the sample to exceed. --optimize= : find parameters to maximize compression ratio given parameters Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints cSpeed= : Minimum compression speed dSpeed= : Minimum decompression speed cMem= : Maximum compression memory - lvl= : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed, + lvl= : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed, stc= : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 100%) : In normal operation, represents percentage slack in choosing viable starting strategy selection in choosing the default parameters (Lower value will begin with stronger strategies) (Default 90%) @@ -121,13 +121,13 @@ Full list of arguments when determining overall winner (default 5 (1% ratio = 5% speed)). tries= : Maximum number of random restarts on a single strategy before switching (Default 5) Higher values will make optimizer run longer, more chances to find better solution. - memLog : Limits the log of the size of each memotable (1 per strategy). Will use hash tables when state space is larger than max size. - Setting memLog = 0 turns off memoization + memLog : Limits the log of the size of each memotable (1 per strategy). Will use hash tables when state space is larger than max size. + Setting memLog = 0 turns off memoization --display= : specifiy which parameters are included in the output - can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters + can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters (Default: display all params available) -P# : generated sample compressibility (when no file is provided) - -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) + -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) -v : Prints Benchmarking output -D : Next argument dictionary file -s : Benchmark all files separately diff --git a/tests/datagencli.c b/tests/datagencli.c index 4814974e27f5..6c1dd4719edc 100644 --- a/tests/datagencli.c +++ b/tests/datagencli.c @@ -121,7 +121,7 @@ int main(int argc, const char** argv) DISPLAYLEVEL(4, "Compressible data Generator \n"); if (probaU32!=COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", probaU32); - DISPLAYLEVEL(3, "Seed = %u \n", seed); + DISPLAYLEVEL(3, "Seed = %u \n", (unsigned)seed); RDG_genStdout(size, (double)probaU32/100, litProba, seed); DISPLAYLEVEL(1, "\n"); diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 2c2276004a95..17c2c18991bc 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -22,7 +22,7 @@ #define ZDICT_STATIC_LINKING_ONLY #include "zdict.h" -// Direct access to internal compression functions is required +/* Direct access to internal compression functions is required */ #include "zstd_compress.c" #define XXH_STATIC_LINKING_ONLY @@ -72,7 +72,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; /*-******************************************************* * Random function *********************************************************/ -static unsigned RAND(unsigned* src) +static U32 RAND(U32* src) { #define RAND_rotl32(x,r) ((x << r) | (x >> (32 - r))) static const U32 prime1 = 2654435761U; @@ -350,7 +350,7 @@ static void writeFrameHeader(U32* seed, frame_t* frame, dictInfo info) } } - DISPLAYLEVEL(3, " frame content size:\t%u\n", (U32)fh.contentSize); + DISPLAYLEVEL(3, " frame content size:\t%u\n", (unsigned)fh.contentSize); DISPLAYLEVEL(3, " frame window size:\t%u\n", fh.windowSize); DISPLAYLEVEL(3, " content size flag:\t%d\n", contentSizeFlag); DISPLAYLEVEL(3, " single segment flag:\t%d\n", singleSegment); @@ -412,7 +412,7 @@ static size_t writeLiteralsBlockSimple(U32* seed, frame_t* frame, size_t content /* RLE literals */ BYTE const symb = (BYTE) (RAND(seed) % 256); - DISPLAYLEVEL(4, " rle literals: 0x%02x\n", (U32)symb); + DISPLAYLEVEL(4, " rle literals: 0x%02x\n", (unsigned)symb); memset(LITERAL_BUFFER, symb, litSize); op[0] = symb; @@ -432,12 +432,12 @@ static size_t writeHufHeader(U32* seed, HUF_CElt* hufTable, void* dst, size_t ds BYTE* op = ostart; unsigned huffLog = 11; - U32 maxSymbolValue = 255; + unsigned maxSymbolValue = 255; - U32 count[HUF_SYMBOLVALUE_MAX+1]; + unsigned count[HUF_SYMBOLVALUE_MAX+1]; /* Scan input and build symbol stats */ - { size_t const largest = HIST_count_wksp (count, &maxSymbolValue, (const BYTE*)src, srcSize, WKSP); + { size_t const largest = HIST_count_wksp (count, &maxSymbolValue, (const BYTE*)src, srcSize, WKSP, sizeof(WKSP)); assert(!HIST_isError(largest)); if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 0; } /* single symbol, rle */ if (largest <= (srcSize >> 7)+1) return 0; /* Fast heuristic : not compressible enough */ @@ -568,8 +568,8 @@ static size_t writeLiteralsBlockCompressed(U32* seed, frame_t* frame, size_t con op += compressedSize; compressedSize += hufHeaderSize; - DISPLAYLEVEL(5, " regenerated size: %u\n", (U32)litSize); - DISPLAYLEVEL(5, " compressed size: %u\n", (U32)compressedSize); + DISPLAYLEVEL(5, " regenerated size: %u\n", (unsigned)litSize); + DISPLAYLEVEL(5, " compressed size: %u\n", (unsigned)compressedSize); if (compressedSize >= litSize) { DISPLAYLEVEL(5, " trying again\n"); /* if we have to try again, reset the stats so we don't accidentally @@ -656,7 +656,7 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, excessMatch = remainingMatch - numSequences * MIN_SEQ_LEN; } - DISPLAYLEVEL(5, " total match lengths: %u\n", (U32)remainingMatch); + DISPLAYLEVEL(5, " total match lengths: %u\n", (unsigned)remainingMatch); for (i = 0; i < numSequences; i++) { /* Generate match and literal lengths by exponential distribution to * ensure nice numbers */ @@ -748,12 +748,13 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, frame->stats.rep[0] = offset; } - DISPLAYLEVEL(6, " LL: %5u OF: %5u ML: %5u", literalLen, offset, matchLen); + DISPLAYLEVEL(6, " LL: %5u OF: %5u ML: %5u", + (unsigned)literalLen, (unsigned)offset, (unsigned)matchLen); DISPLAYLEVEL(7, " srcPos: %8u seqNb: %3u", - (U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart), i); + (unsigned)((BYTE*)srcPtr - (BYTE*)frame->srcStart), (unsigned)i); DISPLAYLEVEL(6, "\n"); if (offsetCode < 3) { - DISPLAYLEVEL(7, " repeat offset: %d\n", repIndex); + DISPLAYLEVEL(7, " repeat offset: %d\n", (int)repIndex); } /* use libzstd sequence handling */ ZSTD_storeSeq(seqStore, literalLen, literals, offsetCode, @@ -766,8 +767,8 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, memcpy(srcPtr, literals, literalsSize); srcPtr += literalsSize; - DISPLAYLEVEL(6, " excess literals: %5u", (U32)literalsSize); - DISPLAYLEVEL(7, " srcPos: %8u", (U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart)); + DISPLAYLEVEL(6, " excess literals: %5u", (unsigned)literalsSize); + DISPLAYLEVEL(7, " srcPos: %8u", (unsigned)((BYTE*)srcPtr - (BYTE*)frame->srcStart)); DISPLAYLEVEL(6, "\n"); return numSequences; @@ -800,7 +801,7 @@ static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr, size_t nbSeq) { /* This code is mostly copied from ZSTD_compressSequences in zstd_compress.c */ - U32 count[MaxSeq+1]; + unsigned count[MaxSeq+1]; S16 norm[MaxSeq+1]; FSE_CTable* CTable_LitLength = frame->stats.litlengthCTable; FSE_CTable* CTable_OffsetBits = frame->stats.offcodeCTable; @@ -823,21 +824,20 @@ static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr, else if (nbSeq < LONGNBSEQ) op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; else op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; - /* seqHead : flags for FSE encoding type */ - seqHead = op++; - if (nbSeq==0) { frame->data = op; - return 0; } + /* seqHead : flags for FSE encoding type */ + seqHead = op++; + /* convert length/distances into codes */ ZSTD_seqToCodes(seqStorePtr); /* CTable for Literal Lengths */ - { U32 max = MaxLL; - size_t const mostFrequent = HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, WKSP); /* cannot fail */ + { unsigned max = MaxLL; + size_t const mostFrequent = HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, WKSP, sizeof(WKSP)); /* cannot fail */ assert(!HIST_isError(mostFrequent)); if (mostFrequent == nbSeq) { /* do RLE if we have the chance */ @@ -868,8 +868,8 @@ static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr, /* CTable for Offsets */ /* see Literal Lengths for descriptions of mode choices */ - { U32 max = MaxOff; - size_t const mostFrequent = HIST_countFast_wksp(count, &max, ofCodeTable, nbSeq, WKSP); /* cannot fail */ + { unsigned max = MaxOff; + size_t const mostFrequent = HIST_countFast_wksp(count, &max, ofCodeTable, nbSeq, WKSP, sizeof(WKSP)); /* cannot fail */ assert(!HIST_isError(mostFrequent)); if (mostFrequent == nbSeq) { *op++ = ofCodeTable[0]; @@ -896,8 +896,8 @@ static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr, /* CTable for MatchLengths */ /* see Literal Lengths for descriptions of mode choices */ - { U32 max = MaxML; - size_t const mostFrequent = HIST_countFast_wksp(count, &max, mlCodeTable, nbSeq, WKSP); /* cannot fail */ + { unsigned max = MaxML; + size_t const mostFrequent = HIST_countFast_wksp(count, &max, mlCodeTable, nbSeq, WKSP, sizeof(WKSP)); /* cannot fail */ assert(!HIST_isError(mostFrequent)); if (mostFrequent == nbSeq) { *op++ = *mlCodeTable; @@ -928,7 +928,7 @@ static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr, initSymbolSet(ofCodeTable, nbSeq, frame->stats.offsetSymbolSet, 28); initSymbolSet(mlCodeTable, nbSeq, frame->stats.matchlengthSymbolSet, 52); - DISPLAYLEVEL(5, " LL type: %d OF type: %d ML type: %d\n", LLtype, Offtype, MLtype); + DISPLAYLEVEL(5, " LL type: %d OF type: %d ML type: %d\n", (unsigned)LLtype, (unsigned)Offtype, (unsigned)MLtype); *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); @@ -1015,11 +1015,11 @@ static size_t writeCompressedBlock(U32* seed, frame_t* frame, size_t contentSize literalsSize = writeLiteralsBlock(seed, frame, contentSize); - DISPLAYLEVEL(4, " literals size: %u\n", (U32)literalsSize); + DISPLAYLEVEL(4, " literals size: %u\n", (unsigned)literalsSize); nbSeq = writeSequencesBlock(seed, frame, contentSize, literalsSize, info); - DISPLAYLEVEL(4, " number of sequences: %u\n", (U32)nbSeq); + DISPLAYLEVEL(4, " number of sequences: %u\n", (unsigned)nbSeq); return (BYTE*)frame->data - blockStart; } @@ -1035,7 +1035,7 @@ static void writeBlock(U32* seed, frame_t* frame, size_t contentSize, BYTE *op = header + 3; DISPLAYLEVEL(4, " block:\n"); - DISPLAYLEVEL(4, " block content size: %u\n", (U32)contentSize); + DISPLAYLEVEL(4, " block content size: %u\n", (unsigned)contentSize); DISPLAYLEVEL(4, " last block: %s\n", lastBlock ? "yes" : "no"); if (blockTypeDesc == 0) { @@ -1083,7 +1083,7 @@ static void writeBlock(U32* seed, frame_t* frame, size_t contentSize, frame->src = (BYTE*)frame->src + contentSize; DISPLAYLEVEL(4, " block type: %s\n", BLOCK_TYPES[blockType]); - DISPLAYLEVEL(4, " block size field: %u\n", (U32)blockSize); + DISPLAYLEVEL(4, " block size field: %u\n", (unsigned)blockSize); header[0] = (BYTE) ((lastBlock | (blockType << 1) | (blockSize << 3)) & 0xff); MEM_writeLE16(header + 1, (U16) (blockSize >> 5)); @@ -1125,7 +1125,7 @@ static void writeChecksum(frame_t* frame) { /* write checksum so implementations can verify their output */ U64 digest = XXH64(frame->srcStart, (BYTE*)frame->src-(BYTE*)frame->srcStart, 0); - DISPLAYLEVEL(3, " checksum: %08x\n", (U32)digest); + DISPLAYLEVEL(3, " checksum: %08x\n", (unsigned)digest); MEM_writeLE32(frame->data, (U32)digest); frame->data = (BYTE*)frame->data + 4; } @@ -1186,7 +1186,7 @@ static U32 generateCompressedBlock(U32 seed, frame_t* frame, dictInfo info) size_t blockContentSize; int blockWritten = 0; BYTE* op; - DISPLAYLEVEL(4, "block seed: %u\n", seed); + DISPLAYLEVEL(4, "block seed: %u\n", (unsigned)seed); initFrame(frame); op = (BYTE*)frame->data; @@ -1223,7 +1223,7 @@ static U32 generateCompressedBlock(U32 seed, frame_t* frame, dictInfo info) DISPLAYLEVEL(5, " can't compress block : try again \n"); } else { blockWritten = 1; - DISPLAYLEVEL(4, " block size: %u \n", (U32)cSize); + DISPLAYLEVEL(4, " block size: %u \n", (unsigned)cSize); frame->src = (BYTE*)frame->src + blockContentSize; } } @@ -1234,7 +1234,7 @@ static U32 generateCompressedBlock(U32 seed, frame_t* frame, dictInfo info) static U32 generateFrame(U32 seed, frame_t* fr, dictInfo info) { /* generate a complete frame */ - DISPLAYLEVEL(3, "frame seed: %u\n", seed); + DISPLAYLEVEL(3, "frame seed: %u\n", (unsigned)seed); initFrame(fr); writeFrameHeader(&seed, fr, info); @@ -1480,8 +1480,8 @@ static int runBlockTest(U32* seed) { size_t const r = testDecodeRawBlock(&fr); if (ZSTD_isError(r)) { - DISPLAY("Error in block mode on test seed %u: %s\n", seedCopy, - ZSTD_getErrorName(r)); + DISPLAY("Error in block mode on test seed %u: %s\n", + (unsigned)seedCopy, ZSTD_getErrorName(r)); return 1; } } @@ -1489,7 +1489,7 @@ static int runBlockTest(U32* seed) { size_t const r = testDecodeWithDict(*seed, gt_block); if (ZSTD_isError(r)) { DISPLAY("Error in block mode with dictionary on test seed %u: %s\n", - seedCopy, ZSTD_getErrorName(r)); + (unsigned)seedCopy, ZSTD_getErrorName(r)); return 1; } } @@ -1507,21 +1507,21 @@ static int runFrameTest(U32* seed) { size_t const r = testDecodeSimple(&fr); if (ZSTD_isError(r)) { DISPLAY("Error in simple mode on test seed %u: %s\n", - seedCopy, ZSTD_getErrorName(r)); + (unsigned)seedCopy, ZSTD_getErrorName(r)); return 1; } } { size_t const r = testDecodeStreaming(&fr); if (ZSTD_isError(r)) { DISPLAY("Error in streaming mode on test seed %u: %s\n", - seedCopy, ZSTD_getErrorName(r)); + (unsigned)seedCopy, ZSTD_getErrorName(r)); return 1; } } { size_t const r = testDecodeWithDict(*seed, gt_frame); /* avoid big dictionaries */ if (ZSTD_isError(r)) { DISPLAY("Error in dictionary mode on test seed %u: %s\n", - seedCopy, ZSTD_getErrorName(r)); + (unsigned)seedCopy, ZSTD_getErrorName(r)); return 1; } } @@ -1538,7 +1538,7 @@ static int runTestMode(U32 seed, unsigned numFiles, unsigned const testDurationS if (numFiles == 0 && !testDurationS) numFiles = 1; - DISPLAY("seed: %u\n", seed); + DISPLAY("seed: %u\n", (unsigned)seed); for (fnum = 0; fnum < numFiles || UTIL_clockSpanMicro(startClock) < maxClockSpan; fnum++) { if (fnum < numFiles) @@ -1568,7 +1568,7 @@ static int generateFile(U32 seed, const char* const path, { frame_t fr; - DISPLAY("seed: %u\n", seed); + DISPLAY("seed: %u\n", (unsigned)seed); { dictInfo const info = initDictInfo(0, 0, NULL, 0); if (genType == gt_frame) { @@ -1590,7 +1590,7 @@ static int generateCorpus(U32 seed, unsigned numFiles, const char* const path, char outPath[MAX_PATH]; unsigned fnum; - DISPLAY("seed: %u\n", seed); + DISPLAY("seed: %u\n", (unsigned)seed); for (fnum = 0; fnum < numFiles; fnum++) { frame_t fr; diff --git a/tests/fullbench.c b/tests/fullbench.c index b05f1537cd70..8644a2e3a136 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -19,7 +19,7 @@ #include "mem.h" /* U32 */ #ifndef ZSTD_DLL_IMPORT - #include "zstd_internal.h" /* ZSTD_blockHeaderSize, blockType_e, KB, MB */ + #include "zstd_internal.h" /* ZSTD_decodeSeqHeaders, ZSTD_blockHeaderSize, blockType_e, KB, MB */ #else #define KB *(1 <<10) #define MB *(1 <<20) @@ -30,7 +30,8 @@ #include "zstd.h" /* ZSTD_versionString */ #include "util.h" /* time functions */ #include "datagen.h" -#include "bench.h" /* CustomBench*/ +#include "benchfn.h" /* CustomBench*/ +#include "benchzstd.h" /* MB_UNIT */ /*_************************************ @@ -63,10 +64,10 @@ static const size_t g_sampleSize = 10000000; /*_************************************ * Benchmark Parameters **************************************/ -static U32 g_nbIterations = NBLOOPS; +static unsigned g_nbIterations = NBLOOPS; static double g_compressibility = COMPRESSIBILITY_DEFAULT; -static void BMK_SetNbIterations(U32 nbLoops) +static void BMK_SetNbIterations(unsigned nbLoops) { g_nbIterations = nbLoops; DISPLAY("- %i iterations -\n", g_nbIterations); @@ -133,7 +134,6 @@ static size_t local_ZSTD_decodeLiteralsBlock(const void* src, size_t srcSize, vo return ZSTD_decodeLiteralsBlock((ZSTD_DCtx*)g_zdc, buff2, g_cSize); } -extern size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeq, const void* src, size_t srcSize); static size_t local_ZSTD_decodeSeqHeaders(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2) { int nbSeq; @@ -171,17 +171,8 @@ local_ZSTD_compress_generic_end(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2) { - ZSTD_outBuffer buffOut; - ZSTD_inBuffer buffIn; (void)buff2; - buffOut.dst = dst; - buffOut.size = dstCapacity; - buffOut.pos = 0; - buffIn.src = src; - buffIn.size = srcSize; - buffIn.pos = 0; - ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end); - return buffOut.pos; + return ZSTD_compress2(g_cstream, dst, dstCapacity, src, srcSize); } static size_t @@ -198,8 +189,8 @@ local_ZSTD_compress_generic_continue(const void* src, size_t srcSize, buffIn.src = src; buffIn.size = srcSize; buffIn.pos = 0; - ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_continue); - ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end); + ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_continue); + ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_end); return buffOut.pos; } @@ -208,18 +199,9 @@ local_ZSTD_compress_generic_T2_end(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2) { - ZSTD_outBuffer buffOut; - ZSTD_inBuffer buffIn; (void)buff2; - ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_nbWorkers, 2); - buffOut.dst = dst; - buffOut.size = dstCapacity; - buffOut.pos = 0; - buffIn.src = src; - buffIn.size = srcSize; - buffIn.pos = 0; - while (ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end)) {} - return buffOut.pos; + ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_nbWorkers, 2); + return ZSTD_compress2(g_cstream, dst, dstCapacity, src, srcSize); } static size_t @@ -230,15 +212,15 @@ local_ZSTD_compress_generic_T2_continue(const void* src, size_t srcSize, ZSTD_outBuffer buffOut; ZSTD_inBuffer buffIn; (void)buff2; - ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_nbWorkers, 2); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_nbWorkers, 2); buffOut.dst = dst; buffOut.size = dstCapacity; buffOut.pos = 0; buffIn.src = src; buffIn.size = srcSize; buffIn.pos = 0; - ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_continue); - while(ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end)) {} + ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_continue); + while(ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_end)) {} return buffOut.pos; } @@ -334,7 +316,7 @@ static size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize, /*_******************************************************* * Bench functions *********************************************************/ -static size_t benchMem(U32 benchNb, +static size_t benchMem(unsigned benchNb, const void* src, size_t srcSize, int cLevel, ZSTD_compressionParameters cparams) { @@ -408,28 +390,28 @@ static size_t benchMem(U32 benchNb, if (g_cstream==NULL) g_cstream = ZSTD_createCStream(); if (g_dstream==NULL) g_dstream = ZSTD_createDStream(); - /* DISPLAY("params: cLevel %d, wlog %d hlog %d clog %d slog %d slen %d tlen %d strat %d \n", + /* DISPLAY("params: cLevel %d, wlog %d hlog %d clog %d slog %d mml %d tlen %d strat %d \n", cLevel, cparams->windowLog, cparams->hashLog, cparams->chainLog, cparams->searchLog, - cparams->searchLength, cparams->targetLength, cparams->strategy); */ + cparams->minMatch, cparams->targetLength, cparams->strategy); */ - ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_compressionLevel, cLevel); - ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_windowLog, cparams.windowLog); - ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_hashLog, cparams.hashLog); - ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_chainLog, cparams.chainLog); - ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_searchLog, cparams.searchLog); - ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_minMatch, cparams.searchLength); - ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_targetLength, cparams.targetLength); - ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_compressionStrategy, cparams.strategy); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_compressionLevel, cLevel); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_windowLog, cparams.windowLog); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_hashLog, cparams.hashLog); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_chainLog, cparams.chainLog); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_searchLog, cparams.searchLog); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_minMatch, cparams.minMatch); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_targetLength, cparams.targetLength); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_strategy, cparams.strategy); - ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, cLevel); - ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_windowLog, cparams.windowLog); - ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_hashLog, cparams.hashLog); - ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_chainLog, cparams.chainLog); - ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_searchLog, cparams.searchLog); - ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_minMatch, cparams.searchLength); - ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_targetLength, cparams.targetLength); - ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionStrategy, cparams.strategy); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_compressionLevel, cLevel); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_windowLog, cparams.windowLog); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_hashLog, cparams.hashLog); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_chainLog, cparams.chainLog); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_searchLog, cparams.searchLog); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_minMatch, cparams.minMatch); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_targetLength, cparams.targetLength); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_strategy, cparams.strategy); /* Preparation */ switch(benchNb) @@ -455,8 +437,8 @@ static size_t benchMem(U32 benchNb, ZSTD_frameHeader zfp; size_t frameHeaderSize, skippedSize; g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel); - frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min); - if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min; + frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_FRAMEHEADERSIZE_MIN); + if (frameHeaderSize==0) frameHeaderSize = ZSTD_FRAMEHEADERSIZE_MIN; ZSTD_getcBlockSize(dstBuff+frameHeaderSize, dstBuffSize, &bp); /* Get 1st block type */ if (bp.blockType != bt_compressed) { DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n"); @@ -476,8 +458,8 @@ static size_t benchMem(U32 benchNb, size_t frameHeaderSize, cBlockSize; ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel); /* it would be better to use direct block compression here */ g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel); - frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min); - if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min; + frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_FRAMEHEADERSIZE_MIN); + if (frameHeaderSize==0) frameHeaderSize = ZSTD_FRAMEHEADERSIZE_MIN; ip += frameHeaderSize; /* Skip frame Header */ cBlockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */ if (bp.blockType != bt_compressed) { @@ -515,20 +497,28 @@ static size_t benchMem(U32 benchNb, /* benchmark loop */ { BMK_timedFnState_t* const tfs = BMK_createTimedFnState(g_nbIterations * 1000, 1000); + void* const avoidStrictAliasingPtr = &dstBuff; + BMK_benchParams_t bp; BMK_runTime_t bestResult; bestResult.sumOfReturn = 0; bestResult.nanoSecPerRun = (unsigned long long)(-1LL); assert(tfs != NULL); + + bp.benchFn = benchFunction; + bp.benchPayload = buff2; + bp.initFn = NULL; + bp.initPayload = NULL; + bp.errorFn = ZSTD_isError; + bp.blockCount = 1; + bp.srcBuffers = &src; + bp.srcSizes = &srcSize; + bp.dstBuffers = (void* const*) avoidStrictAliasingPtr; /* circumvent strict aliasing warning on gcc-8, + * because gcc considers that `void* const *` and `void**` are 2 different types */ + bp.dstCapacities = &dstBuffSize; + bp.blockResults = NULL; + for (;;) { - void* const dstBuffv = dstBuff; - BMK_runOutcome_t const bOutcome = - BMK_benchTimedFn( tfs, - benchFunction, buff2, - NULL, NULL, /* initFn */ - 1, /* blockCount */ - &src, &srcSize, - &dstBuffv, &dstBuffSize, - NULL); + BMK_runOutcome_t const bOutcome = BMK_benchTimedFn(tfs, bp); if (!BMK_isSuccessful_runOutcome(bOutcome)) { DISPLAY("ERROR benchmarking function ! ! \n"); @@ -616,7 +606,7 @@ static int benchFiles(U32 benchNb, benchedSize = (size_t)inFileSize; if ((U64)benchedSize < inFileSize) { DISPLAY("Not enough memory for '%s' full size; testing %u MB only... \n", - inFileName, (U32)(benchedSize>>20)); + inFileName, (unsigned)(benchedSize>>20)); } } /* Alloc */ @@ -744,7 +734,7 @@ int main(int argc, const char** argv) if (longCommandWArg(&argument, "chainLog=") || longCommandWArg(&argument, "clog=")) { cparams.chainLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "hashLog=") || longCommandWArg(&argument, "hlog=")) { cparams.hashLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "searchLog=") || longCommandWArg(&argument, "slog=")) { cparams.searchLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "searchLength=") || longCommandWArg(&argument, "slen=")) { cparams.searchLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "minMatch=") || longCommandWArg(&argument, "mml=")) { cparams.minMatch = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "targetLength=") || longCommandWArg(&argument, "tlen=")) { cparams.targetLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { cparams.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevel = (int)readU32FromChar(&argument); cparams = ZSTD_getCParams(cLevel, 0, 0); if (argument[0]==',') { argument++; continue; } else break; } diff --git a/tests/fuzz/Makefile b/tests/fuzz/Makefile index d9b00fd2af79..4130f18e330c 100644 --- a/tests/fuzz/Makefile +++ b/tests/fuzz/Makefile @@ -41,7 +41,7 @@ FUZZ_ARFLAGS := $(ARFLAGS) FUZZ_TARGET_FLAGS = $(FUZZ_CPPFLAGS) $(FUZZ_CXXFLAGS) $(FUZZ_LDFLAGS) FUZZ_HEADERS := fuzz_helpers.h fuzz.h zstd_helpers.h -FUZZ_SRC := zstd_helpers.c +FUZZ_SRC := $(PRGDIR)/util.c zstd_helpers.c ZSTDCOMMON_SRC := $(ZSTDDIR)/common/*.c ZSTDCOMP_SRC := $(ZSTDDIR)/compress/*.c @@ -90,7 +90,7 @@ stream_decompress: $(FUZZ_HEADERS) $(FUZZ_OBJ) stream_decompress.o block_decompress: $(FUZZ_HEADERS) $(FUZZ_OBJ) block_decompress.o $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) block_decompress.o $(LIB_FUZZING_ENGINE) -o $@ -libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h regression_driver.o +libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h $(PRGDIR)/util.c regression_driver.o $(AR) $(FUZZ_ARFLAGS) $@ regression_driver.o # Install libfuzzer (not usable for MSAN testing) @@ -99,7 +99,7 @@ libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h regression_driver.o .PHONY: libFuzzer libFuzzer: @$(RM) -rf Fuzzer - @git clone https://chromium.googlesource.com/chromium/llvm-project/llvm/lib/Fuzzer + @git clone https://chromium.googlesource.com/chromium/llvm-project/compiler-rt/lib/fuzzer Fuzzer @cd Fuzzer && ./build.sh corpora/%_seed_corpus.zip: diff --git a/tests/fuzz/simple_round_trip.c b/tests/fuzz/simple_round_trip.c index 0921106dea8e..83608b6e79ea 100644 --- a/tests/fuzz/simple_round_trip.c +++ b/tests/fuzz/simple_round_trip.c @@ -40,9 +40,9 @@ static size_t roundTripTest(void *result, size_t resultCapacity, ZSTD_outBuffer out = {compressed, compressedCapacity, 0}; size_t err; - ZSTD_CCtx_reset(cctx); + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); FUZZ_setRandomParameters(cctx, srcSize, &seed); - err = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end); + err = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end); FUZZ_ZASSERT(err); FUZZ_ASSERT(err == 0); cSize = out.pos; diff --git a/tests/fuzz/stream_round_trip.c b/tests/fuzz/stream_round_trip.c index 72d70495f836..d903bcb29b11 100644 --- a/tests/fuzz/stream_round_trip.c +++ b/tests/fuzz/stream_round_trip.c @@ -56,7 +56,7 @@ static size_t compress(uint8_t *dst, size_t capacity, const uint8_t *src, size_t srcSize) { size_t dstSize = 0; - ZSTD_CCtx_reset(cctx); + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); FUZZ_setRandomParameters(cctx, srcSize, &seed); while (srcSize > 0) { @@ -72,7 +72,7 @@ static size_t compress(uint8_t *dst, size_t capacity, case 1: /* fall-though */ case 2: { size_t const ret = - ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_flush); + ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush); FUZZ_ZASSERT(ret); if (ret == 0) mode = -1; @@ -80,11 +80,11 @@ static size_t compress(uint8_t *dst, size_t capacity, } case 3: { size_t ret = - ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end); + ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end); FUZZ_ZASSERT(ret); /* Reset the compressor when the frame is finished */ if (ret == 0) { - ZSTD_CCtx_reset(cctx); + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); if ((FUZZ_rand(&seed) & 7) == 0) { size_t const remaining = in.size - in.pos; FUZZ_setRandomParameters(cctx, remaining, &seed); @@ -95,7 +95,7 @@ static size_t compress(uint8_t *dst, size_t capacity, } default: { size_t const ret = - ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_continue); + ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue); FUZZ_ZASSERT(ret); mode = -1; } @@ -108,7 +108,7 @@ static size_t compress(uint8_t *dst, size_t capacity, for (;;) { ZSTD_inBuffer in = {NULL, 0, 0}; ZSTD_outBuffer out = makeOutBuffer(dst, capacity); - size_t const ret = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end); + size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end); FUZZ_ZASSERT(ret); dst += out.pos; diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c index bf5eccff83c7..10163e1512b0 100644 --- a/tests/fuzz/zstd_helpers.c +++ b/tests/fuzz/zstd_helpers.c @@ -13,7 +13,7 @@ #include "fuzz_helpers.h" #include "zstd.h" -static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned value) +static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, int value) { FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, param, value)); } @@ -32,10 +32,10 @@ ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state) cParams.hashLog = FUZZ_rand32(state, ZSTD_HASHLOG_MIN, 15); cParams.chainLog = FUZZ_rand32(state, ZSTD_CHAINLOG_MIN, 16); cParams.searchLog = FUZZ_rand32(state, ZSTD_SEARCHLOG_MIN, 9); - cParams.searchLength = FUZZ_rand32(state, ZSTD_SEARCHLENGTH_MIN, - ZSTD_SEARCHLENGTH_MAX); + cParams.minMatch = FUZZ_rand32(state, ZSTD_MINMATCH_MIN, + ZSTD_MINMATCH_MAX); cParams.targetLength = FUZZ_rand32(state, 0, 512); - cParams.strategy = FUZZ_rand32(state, ZSTD_fast, ZSTD_btultra); + cParams.strategy = FUZZ_rand32(state, ZSTD_STRATEGY_MIN, ZSTD_STRATEGY_MAX); return ZSTD_adjustCParams(cParams, srcSize, 0); } @@ -60,25 +60,25 @@ ZSTD_parameters FUZZ_randomParams(size_t srcSize, uint32_t *state) void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state) { ZSTD_compressionParameters cParams = FUZZ_randomCParams(srcSize, state); - set(cctx, ZSTD_p_windowLog, cParams.windowLog); - set(cctx, ZSTD_p_hashLog, cParams.hashLog); - set(cctx, ZSTD_p_chainLog, cParams.chainLog); - set(cctx, ZSTD_p_searchLog, cParams.searchLog); - set(cctx, ZSTD_p_minMatch, cParams.searchLength); - set(cctx, ZSTD_p_targetLength, cParams.targetLength); - set(cctx, ZSTD_p_compressionStrategy, cParams.strategy); + set(cctx, ZSTD_c_windowLog, cParams.windowLog); + set(cctx, ZSTD_c_hashLog, cParams.hashLog); + set(cctx, ZSTD_c_chainLog, cParams.chainLog); + set(cctx, ZSTD_c_searchLog, cParams.searchLog); + set(cctx, ZSTD_c_minMatch, cParams.minMatch); + set(cctx, ZSTD_c_targetLength, cParams.targetLength); + set(cctx, ZSTD_c_strategy, cParams.strategy); /* Select frame parameters */ - setRand(cctx, ZSTD_p_contentSizeFlag, 0, 1, state); - setRand(cctx, ZSTD_p_checksumFlag, 0, 1, state); - setRand(cctx, ZSTD_p_dictIDFlag, 0, 1, state); - setRand(cctx, ZSTD_p_forceAttachDict, -2, 2, state); + setRand(cctx, ZSTD_c_contentSizeFlag, 0, 1, state); + setRand(cctx, ZSTD_c_checksumFlag, 0, 1, state); + setRand(cctx, ZSTD_c_dictIDFlag, 0, 1, state); + setRand(cctx, ZSTD_c_forceAttachDict, 0, 2, state); /* Select long distance matchig parameters */ - setRand(cctx, ZSTD_p_enableLongDistanceMatching, 0, 1, state); - setRand(cctx, ZSTD_p_ldmHashLog, ZSTD_HASHLOG_MIN, 16, state); - setRand(cctx, ZSTD_p_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN, + setRand(cctx, ZSTD_c_enableLongDistanceMatching, 0, 1, state); + setRand(cctx, ZSTD_c_ldmHashLog, ZSTD_HASHLOG_MIN, 16, state); + setRand(cctx, ZSTD_c_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX, state); - setRand(cctx, ZSTD_p_ldmBucketSizeLog, 0, ZSTD_LDM_BUCKETSIZELOG_MAX, + setRand(cctx, ZSTD_c_ldmBucketSizeLog, 0, ZSTD_LDM_BUCKETSIZELOG_MAX, state); - setRand(cctx, ZSTD_p_ldmHashEveryLog, 0, - ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN, state); + setRand(cctx, ZSTD_c_ldmHashRateLog, ZSTD_LDM_HASHRATELOG_MIN, + ZSTD_LDM_HASHRATELOG_MAX, state); } diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 5616285b9ed7..946844e1614a 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -47,8 +47,8 @@ #define MB *(1U<<20) #define GB *(1U<<30) -static const U32 FUZ_compressibility_default = 50; -static const U32 nbTestsDefault = 30000; +static const int FUZ_compressibility_default = 50; +static const int nbTestsDefault = 30000; /*-************************************ @@ -88,7 +88,7 @@ void FUZ_bug976(void) #define MAX(a,b) ((a)>(b)?(a):(b)) #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) -static unsigned FUZ_rand(unsigned* src) +static U32 FUZ_rand(U32* src) { static const U32 prime1 = 2654435761U; static const U32 prime2 = 2246822519U; @@ -100,7 +100,7 @@ static unsigned FUZ_rand(unsigned* src) return rand32 >> 5; } -static unsigned FUZ_highbit32(U32 v32) +static U32 FUZ_highbit32(U32 v32) { unsigned nbBits = 0; if (v32==0) return 0; @@ -155,7 +155,7 @@ static void* FUZ_mallocDebug(void* counter, size_t size) void* const ptr = malloc(size); if (ptr==NULL) return NULL; DISPLAYLEVEL(4, "allocating %u KB => effectively %u KB \n", - (U32)(size >> 10), (U32)(malloc_size(ptr) >> 10)); /* OS-X specific */ + (unsigned)(size >> 10), (unsigned)(malloc_size(ptr) >> 10)); /* OS-X specific */ mcPtr->totalMalloc += size; mcPtr->currentMalloc += size; if (mcPtr->currentMalloc > mcPtr->peakMalloc) @@ -167,7 +167,7 @@ static void* FUZ_mallocDebug(void* counter, size_t size) static void FUZ_freeDebug(void* counter, void* address) { mallocCounter_t* const mcPtr = (mallocCounter_t*)counter; - DISPLAYLEVEL(4, "freeing %u KB \n", (U32)(malloc_size(address) >> 10)); + DISPLAYLEVEL(4, "freeing %u KB \n", (unsigned)(malloc_size(address) >> 10)); mcPtr->nbFree += 1; mcPtr->currentMalloc -= malloc_size(address); /* OS-X specific */ free(address); @@ -176,9 +176,9 @@ static void FUZ_freeDebug(void* counter, void* address) static void FUZ_displayMallocStats(mallocCounter_t count) { DISPLAYLEVEL(3, "peak:%6u KB, nbMallocs:%2u, total:%6u KB \n", - (U32)(count.peakMalloc >> 10), + (unsigned)(count.peakMalloc >> 10), count.nbMalloc, - (U32)(count.totalMalloc >> 10)); + (unsigned)(count.totalMalloc >> 10)); } static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsigned part, @@ -226,18 +226,16 @@ static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsig /* advanced MT API test */ if (part <= 3) - { U32 nbThreads; + { unsigned nbThreads; for (nbThreads=1; nbThreads<=4; nbThreads++) { int compressionLevel; for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { mallocCounter_t malcount = INIT_MALLOC_COUNTER; ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem); - ZSTD_outBuffer out = { outBuffer, outSize, 0 }; - ZSTD_inBuffer in = { inBuffer, inSize, 0 }; - CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, (U32)compressionLevel) ); - CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbWorkers, nbThreads) ); - while ( ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end) ) {} + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) ); + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) ); + CHECK_Z( ZSTD_compress2(cctx, outBuffer, outSize, inBuffer, inSize) ); ZSTD_freeCCtx(cctx); DISPLAYLEVEL(3, "compress_generic,-T%u,end level %i : ", nbThreads, compressionLevel); @@ -246,7 +244,7 @@ static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsig /* advanced MT streaming API test */ if (part <= 4) - { U32 nbThreads; + { unsigned nbThreads; for (nbThreads=1; nbThreads<=4; nbThreads++) { int compressionLevel; for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { @@ -255,10 +253,10 @@ static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsig ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem); ZSTD_outBuffer out = { outBuffer, outSize, 0 }; ZSTD_inBuffer in = { inBuffer, inSize, 0 }; - CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, (U32)compressionLevel) ); - CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbWorkers, nbThreads) ); - CHECK_Z( ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_continue) ); - while ( ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end) ) {} + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) ); + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) ); + CHECK_Z( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue) ); + while ( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) ) {} ZSTD_freeCCtx(cctx); DISPLAYLEVEL(3, "compress_generic,-T%u,continue level %i : ", nbThreads, compressionLevel); @@ -311,9 +309,8 @@ static int basicUnitTests(U32 seed, double compressibility) size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize); void* const compressedBuffer = malloc(compressedBufferSize); void* const decodedBuffer = malloc(CNBuffSize); - ZSTD_DCtx* dctx = ZSTD_createDCtx(); int testResult = 0; - U32 testNb=0; + unsigned testNb=0; size_t cSize; /* Create compressible noise */ @@ -325,37 +322,45 @@ static int basicUnitTests(U32 seed, double compressibility) RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed); /* Basic tests */ - DISPLAYLEVEL(3, "test%3i : ZSTD_getErrorName : ", testNb++); + DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName : ", testNb++); { const char* errorString = ZSTD_getErrorName(0); DISPLAYLEVEL(3, "OK : %s \n", errorString); } - DISPLAYLEVEL(3, "test%3i : ZSTD_getErrorName with wrong value : ", testNb++); + DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName with wrong value : ", testNb++); { const char* errorString = ZSTD_getErrorName(499); DISPLAYLEVEL(3, "OK : %s \n", errorString); } - DISPLAYLEVEL(3, "test%3i : min compression level : ", testNb++); + DISPLAYLEVEL(3, "test%3u : min compression level : ", testNb++); { int const mcl = ZSTD_minCLevel(); DISPLAYLEVEL(3, "%i (OK) \n", mcl); } - DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, (U32)CNBuffSize); + DISPLAYLEVEL(3, "test%3u : compress %u bytes : ", testNb++, (unsigned)CNBuffSize); { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); if (cctx==NULL) goto _output_error; CHECKPLUS(r, ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, 1), cSize=r ); - DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); DISPLAYLEVEL(3, "test%3i : size of cctx for level 1 : ", testNb++); { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx); - DISPLAYLEVEL(3, "%u bytes \n", (U32)cctxSize); + DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cctxSize); } ZSTD_freeCCtx(cctx); } + DISPLAYLEVEL(3, "test%3i : decompress skippable frame -8 size : ", testNb++); + { + char const skippable8[] = "\x50\x2a\x4d\x18\xf8\xff\xff\xff"; + size_t const size = ZSTD_decompress(NULL, 0, skippable8, 8); + if (!ZSTD_isError(size)) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameContentSize test : ", testNb++); { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize); @@ -369,7 +374,7 @@ static int basicUnitTests(U32 seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); - DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (U32)CNBuffSize); + DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize); { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); if (r != CNBuffSize) goto _output_error; } DISPLAYLEVEL(3, "OK \n"); @@ -383,13 +388,27 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "test%3i : decompress with null dict : ", testNb++); - { size_t const r = ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL, 0); - if (r != CNBuffSize) goto _output_error; } + { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); + { size_t const r = ZSTD_decompress_usingDict(dctx, + decodedBuffer, CNBuffSize, + compressedBuffer, cSize, + NULL, 0); + if (r != CNBuffSize) goto _output_error; + } + ZSTD_freeDCtx(dctx); + } DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3i : decompress with null DDict : ", testNb++); - { size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL); - if (r != CNBuffSize) goto _output_error; } + { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); + { size_t const r = ZSTD_decompress_usingDDict(dctx, + decodedBuffer, CNBuffSize, + compressedBuffer, cSize, + NULL); + if (r != CNBuffSize) goto _output_error; + } + ZSTD_freeDCtx(dctx); + } DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3i : decompress with 1 missing byte : ", testNb++); @@ -411,11 +430,55 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3d : check CCtx size after compressing empty input : ", testNb++); - { ZSTD_CCtx* cctx = ZSTD_createCCtx(); + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); size_t const r = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 19); if (ZSTD_isError(r)) goto _output_error; if (ZSTD_sizeof_CCtx(cctx) > (1U << 20)) goto _output_error; ZSTD_freeCCtx(cctx); + cSize = r; + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3d : decompress empty frame into NULL : ", testNb++); + { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, cSize); + if (ZSTD_isError(r)) goto _output_error; + if (r != 0) goto _output_error; + } + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_outBuffer output; + if (cctx==NULL) goto _output_error; + output.dst = compressedBuffer; + output.size = compressedBufferSize; + output.pos = 0; + CHECK_Z( ZSTD_initCStream(cctx, 1) ); /* content size unknown */ + CHECK_Z( ZSTD_flushStream(cctx, &output) ); /* ensure no possibility to "concatenate" and determine the content size */ + CHECK_Z( ZSTD_endStream(cctx, &output) ); + ZSTD_freeCCtx(cctx); + /* single scan decompression */ + { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, output.pos); + if (ZSTD_isError(r)) goto _output_error; + if (r != 0) goto _output_error; + } + /* streaming decompression */ + { ZSTD_DCtx* const dstream = ZSTD_createDStream(); + ZSTD_inBuffer dinput; + ZSTD_outBuffer doutput; + size_t ipos; + if (dstream==NULL) goto _output_error; + dinput.src = compressedBuffer; + dinput.size = 0; + dinput.pos = 0; + doutput.dst = NULL; + doutput.size = 0; + doutput.pos = 0; + CHECK_Z ( ZSTD_initDStream(dstream) ); + for (ipos=1; ipos<=output.pos; ipos++) { + dinput.size = ipos; + CHECK_Z ( ZSTD_decompressStream(dstream, &doutput, &dinput) ); + } + if (doutput.pos != 0) goto _output_error; + ZSTD_freeDStream(dstream); + } } DISPLAYLEVEL(3, "OK \n"); @@ -437,7 +500,8 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3d : re-using a CCtx should compress the same : ", testNb++); - { int i; + { size_t const sampleSize = 30; + int i; for (i=0; i<20; i++) ((char*)CNBuffer)[i] = (char)i; /* ensure no match during initial section */ memcpy((char*)CNBuffer + 20, CNBuffer, 10); /* create one match, starting from beginning of sample, which is the difficult case (see #1241) */ @@ -445,54 +509,117 @@ static int basicUnitTests(U32 seed, double compressibility) ZSTD_CCtx* const cctx = ZSTD_createCCtx(); size_t size1, size2; DISPLAYLEVEL(5, "l%i ", i); - size1 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 30, i); + size1 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i); CHECK_Z(size1); - size2 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 30, i); + + size2 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i); + CHECK_Z(size2); + CHECK_EQ(size1, size2); + + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, i) ); + size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize); + CHECK_Z(size2); + CHECK_EQ(size1, size2); + + size2 = ZSTD_compress2(cctx, compressedBuffer, ZSTD_compressBound(sampleSize) - 1, CNBuffer, sampleSize); /* force streaming, as output buffer is not large enough to guarantee success */ CHECK_Z(size2); CHECK_EQ(size1, size2); + { ZSTD_inBuffer inb; + ZSTD_outBuffer outb; + inb.src = CNBuffer; + inb.pos = 0; + inb.size = sampleSize; + outb.dst = compressedBuffer; + outb.pos = 0; + outb.size = ZSTD_compressBound(sampleSize) - 1; /* force streaming, as output buffer is not large enough to guarantee success */ + CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); + assert(inb.pos == inb.size); + CHECK_EQ(size1, outb.pos); + } + ZSTD_freeCCtx(cctx); } } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3d : btultra2 & 1st block : ", testNb++); + { size_t const sampleSize = 1024; + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_inBuffer inb; + ZSTD_outBuffer outb; + inb.src = CNBuffer; + inb.pos = 0; + inb.size = 0; + outb.dst = compressedBuffer; + outb.pos = 0; + outb.size = compressedBufferSize; + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, ZSTD_maxCLevel()) ); + + inb.size = sampleSize; /* start with something, so that context is already used */ + CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */ + assert(inb.pos == inb.size); + outb.pos = 0; /* cancel output */ + + CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(cctx, sampleSize) ); + inb.size = 4; /* too small size : compression will be skipped */ + inb.pos = 0; + CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) ); + assert(inb.pos == inb.size); + + inb.size += 5; /* too small size : compression will be skipped */ + CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) ); + assert(inb.pos == inb.size); + + inb.size += 11; /* small enough to attempt compression */ + CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) ); + assert(inb.pos == inb.size); + + assert(inb.pos < sampleSize); + inb.size = sampleSize; /* large enough to trigger stats_init, but no longer at beginning */ + CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */ + assert(inb.pos == inb.size); + ZSTD_freeCCtx(cctx); + } + DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_getParameter() : ", testNb++); { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); ZSTD_outBuffer out = {NULL, 0, 0}; ZSTD_inBuffer in = {NULL, 0, 0}; - unsigned value; + int value; - CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_compressionLevel, &value)); + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); CHECK_EQ(value, 3); - CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_hashLog, &value)); + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); CHECK_EQ(value, 0); - CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_p_hashLog, ZSTD_HASHLOG_MIN)); - CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_compressionLevel, &value)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, ZSTD_HASHLOG_MIN)); + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); CHECK_EQ(value, 3); - CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_hashLog, &value)); + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); CHECK_EQ(value, ZSTD_HASHLOG_MIN); - CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, 7)); - CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_compressionLevel, &value)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7)); + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); CHECK_EQ(value, 7); - CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_hashLog, &value)); + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); CHECK_EQ(value, ZSTD_HASHLOG_MIN); /* Start a compression job */ - ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_continue); - CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_compressionLevel, &value)); + ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue); + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); CHECK_EQ(value, 7); - CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_hashLog, &value)); + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); CHECK_EQ(value, ZSTD_HASHLOG_MIN); /* Reset the CCtx */ - ZSTD_CCtx_reset(cctx); - CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_compressionLevel, &value)); + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); CHECK_EQ(value, 7); - CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_hashLog, &value)); + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); CHECK_EQ(value, ZSTD_HASHLOG_MIN); /* Reset the parameters */ - ZSTD_CCtx_resetParameters(cctx); - CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_compressionLevel, &value)); + ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters); + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); CHECK_EQ(value, 3); - CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_hashLog, &value)); + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); CHECK_EQ(value, 0); ZSTD_freeCCtx(cctx); @@ -502,8 +629,8 @@ static int basicUnitTests(U32 seed, double compressibility) /* this test is really too long, and should be made faster */ DISPLAYLEVEL(3, "test%3d : overflow protection with large windowLog : ", testNb++); { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - ZSTD_parameters params = ZSTD_getParams(-9, ZSTD_CONTENTSIZE_UNKNOWN, 0); - size_t const nbCompressions = ((1U << 31) / CNBuffSize) + 1; /* ensure U32 overflow protection is triggered */ + ZSTD_parameters params = ZSTD_getParams(-999, ZSTD_CONTENTSIZE_UNKNOWN, 0); + size_t const nbCompressions = ((1U << 31) / CNBuffSize) + 2; /* ensure U32 overflow protection is triggered */ size_t cnb; assert(cctx != NULL); params.fParams.contentSizeFlag = 0; @@ -537,7 +664,7 @@ static int basicUnitTests(U32 seed, double compressibility) { U32 const maxNbAttempts = 1100; /* nb of usages before triggering size down is handled within zstd_compress.c. * currently defined as 128x, but could be adjusted in the future. * make this test long enough so that it's not too much tied to the current definition within zstd_compress.c */ - U32 u; + unsigned u; for (u=0; u<maxNbAttempts; u++) { CHECK_Z(ZSTD_compressCCtx(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1)); if (ZSTD_sizeof_CCtx(largeCCtx) < largeCCtxSize) break; /* sized down */ @@ -580,7 +707,7 @@ static int basicUnitTests(U32 seed, double compressibility) CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL), cSize=r ); DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", - (U32)cSize, (double)cSize/CNBuffSize*100); + (unsigned)cSize, (double)cSize/CNBuffSize*100); DISPLAYLEVEL(3, "test%3i : simple decompression test with static DCtx : ", testNb++); { size_t const r = ZSTD_decompressDCtx(staticDCtx, @@ -631,6 +758,17 @@ static int basicUnitTests(U32 seed, double compressibility) free(staticDCtxBuffer); } + DISPLAYLEVEL(3, "test%3i : Static negative levels : ", testNb++); + { size_t const cctxSizeN1 = ZSTD_estimateCCtxSize(-1); + size_t const cctxSizeP1 = ZSTD_estimateCCtxSize(1); + size_t const cstreamSizeN1 = ZSTD_estimateCStreamSize(-1); + size_t const cstreamSizeP1 = ZSTD_estimateCStreamSize(1); + + if (!(0 < cctxSizeN1 && cctxSizeN1 <= cctxSizeP1)) goto _output_error; + if (!(0 < cstreamSizeN1 && cstreamSizeN1 <= cstreamSizeP1)) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + /* ZSTDMT simple MT compression test */ DISPLAYLEVEL(3, "test%3i : create ZSTDMT CCtx : ", testNb++); @@ -642,23 +780,23 @@ static int basicUnitTests(U32 seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); - DISPLAYLEVEL(3, "test%3i : compress %u bytes with 2 threads : ", testNb++, (U32)CNBuffSize); + DISPLAYLEVEL(3, "test%3u : compress %u bytes with 2 threads : ", testNb++, (unsigned)CNBuffSize); CHECKPLUS(r, ZSTDMT_compressCCtx(mtctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, 1), cSize=r ); - DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); DISPLAYLEVEL(3, "test%3i : decompressed size test : ", testNb++); { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize); if (rSize != CNBuffSize) { - DISPLAY("ZSTD_getFrameContentSize incorrect : %u != %u \n", (U32)rSize, (U32)CNBuffSize); + DISPLAY("ZSTD_getFrameContentSize incorrect : %u != %u \n", (unsigned)rSize, (unsigned)CNBuffSize); goto _output_error; } } DISPLAYLEVEL(3, "OK \n"); - DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (U32)CNBuffSize); + DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize); { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); if (r != CNBuffSize) goto _output_error; } DISPLAYLEVEL(3, "OK \n"); @@ -680,9 +818,9 @@ static int basicUnitTests(U32 seed, double compressibility) NULL, params, 3 /*overlapRLog*/), cSize=r ); } - DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); - DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (U32)CNBuffSize); + DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize); { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); if (r != CNBuffSize) goto _output_error; } DISPLAYLEVEL(3, "OK \n"); @@ -709,7 +847,7 @@ static int basicUnitTests(U32 seed, double compressibility) const U32 skipLen = 129 KB; MEM_writeLE32((BYTE*)compressedBuffer + off, ZSTD_MAGIC_SKIPPABLE_START); MEM_writeLE32((BYTE*)compressedBuffer + off + 4, skipLen); - off += skipLen + ZSTD_skippableHeaderSize; + off += skipLen + ZSTD_SKIPPABLEHEADERSIZE; } } cSize = off; @@ -733,7 +871,9 @@ static int basicUnitTests(U32 seed, double compressibility) /* Dictionary and CCtx Duplication tests */ { ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx(); ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); static const size_t dictSize = 551; + assert(dctx != NULL); assert(ctxOrig != NULL); assert(ctxDuplicated != NULL); DISPLAYLEVEL(3, "test%3i : copy context too soon : ", testNb++); { size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0); @@ -750,7 +890,7 @@ static int basicUnitTests(U32 seed, double compressibility) CHECKPLUS(r, ZSTD_compressEnd(ctxOrig, compressedBuffer, compressedBufferSize, (const char*)CNBuffer + dictSize, CNBuffSize - dictSize), cSize += r); - DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); DISPLAYLEVEL(3, "test%3i : frame built with flat dictionary should be decompressible : ", testNb++); CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, @@ -768,7 +908,7 @@ static int basicUnitTests(U32 seed, double compressibility) cSize += r); if (cSize != cSizeOrig) goto _output_error; /* should be identical ==> same size */ } - DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); DISPLAYLEVEL(3, "test%3i : frame built with duplicated context should be decompressible : ", testNb++); CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, @@ -782,7 +922,7 @@ static int basicUnitTests(U32 seed, double compressibility) { ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, dictSize); size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict); if (r != CNBuffSize - dictSize) goto _output_error; - DISPLAYLEVEL(3, "OK (size of DDict : %u) \n", (U32)ZSTD_sizeof_DDict(ddict)); + DISPLAYLEVEL(3, "OK (size of DDict : %u) \n", (unsigned)ZSTD_sizeof_DDict(ddict)); ZSTD_freeDDict(ddict); } @@ -795,7 +935,7 @@ static int basicUnitTests(U32 seed, double compressibility) if (r != CNBuffSize - dictSize) goto _output_error; } free(ddictBuffer); - DISPLAYLEVEL(3, "OK (size of static DDict : %u) \n", (U32)ddictBufferSize); + DISPLAYLEVEL(3, "OK (size of static DDict : %u) \n", (unsigned)ddictBufferSize); } DISPLAYLEVEL(3, "test%3i : check content size on duplicated context : ", testNb++); @@ -817,6 +957,7 @@ static int basicUnitTests(U32 seed, double compressibility) ZSTD_freeCCtx(ctxOrig); ZSTD_freeCCtx(ctxDuplicated); + ZSTD_freeDCtx(dctx); } /* Dictionary and dictBuilder tests */ @@ -843,7 +984,7 @@ static int basicUnitTests(U32 seed, double compressibility) { size_t const sDictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity, decodedBuffer, samplesSizes, nbSamples); if (ZDICT_isError(sDictSize)) goto _output_error; - DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (U32)sDictSize); + DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)sDictSize); } DISPLAYLEVEL(3, "test%3i : dictBuilder : ", testNb++); @@ -851,19 +992,47 @@ static int basicUnitTests(U32 seed, double compressibility) dictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity, CNBuffer, samplesSizes, nbSamples); if (ZDICT_isError(dictSize)) goto _output_error; - DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (U32)dictSize); + DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); + + DISPLAYLEVEL(3, "test%3i : Multithreaded COVER dictBuilder : ", testNb++); + { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } + { ZDICT_cover_params_t coverParams; + memset(&coverParams, 0, sizeof(coverParams)); + coverParams.steps = 8; + coverParams.nbThreads = 4; + dictSize = ZDICT_optimizeTrainFromBuffer_cover( + dictBuffer, dictBufferCapacity, + CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */ + &coverParams); + if (ZDICT_isError(dictSize)) goto _output_error; + } + DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); + + DISPLAYLEVEL(3, "test%3i : Multithreaded FASTCOVER dictBuilder : ", testNb++); + { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } + { ZDICT_fastCover_params_t fastCoverParams; + memset(&fastCoverParams, 0, sizeof(fastCoverParams)); + fastCoverParams.steps = 8; + fastCoverParams.nbThreads = 4; + dictSize = ZDICT_optimizeTrainFromBuffer_fastCover( + dictBuffer, dictBufferCapacity, + CNBuffer, samplesSizes, nbSamples, + &fastCoverParams); + if (ZDICT_isError(dictSize)) goto _output_error; + } + DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++); dictID = ZDICT_getDictID(dictBuffer, dictSize); if (dictID==0) goto _output_error; - DISPLAYLEVEL(3, "OK : %u \n", dictID); + DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID); DISPLAYLEVEL(3, "test%3i : compress with dictionary : ", testNb++); cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, dictBuffer, dictSize, 4); if (ZSTD_isError(cSize)) goto _output_error; - DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); DISPLAYLEVEL(3, "test%3i : retrieve dictID from dictionary : ", testNb++); { U32 const did = ZSTD_getDictID_fromDict(dictBuffer, dictSize); @@ -878,17 +1047,20 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++); - CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, - decodedBuffer, CNBuffSize, - compressedBuffer, cSize, - dictBuffer, dictSize), - if (r != CNBuffSize) goto _output_error); + { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); + CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, + decodedBuffer, CNBuffSize, + compressedBuffer, cSize, + dictBuffer, dictSize), + if (r != CNBuffSize) goto _output_error); + ZSTD_freeDCtx(dctx); + } DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3i : estimate CDict size : ", testNb++); { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); size_t const estimatedSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byRef); - DISPLAYLEVEL(3, "OK : %u \n", (U32)estimatedSize); + DISPLAYLEVEL(3, "OK : %u \n", (unsigned)estimatedSize); } DISPLAYLEVEL(3, "test%3i : compress with CDict ", testNb++); @@ -896,13 +1068,13 @@ static int basicUnitTests(U32 seed, double compressibility) ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); - DISPLAYLEVEL(3, "(size : %u) : ", (U32)ZSTD_sizeof_CDict(cdict)); + DISPLAYLEVEL(3, "(size : %u) : ", (unsigned)ZSTD_sizeof_CDict(cdict)); cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, cdict); ZSTD_freeCDict(cdict); if (ZSTD_isError(cSize)) goto _output_error; } - DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++); { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize); @@ -911,11 +1083,14 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++); - CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, - decodedBuffer, CNBuffSize, - compressedBuffer, cSize, - dictBuffer, dictSize), - if (r != CNBuffSize) goto _output_error); + { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); + CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, + decodedBuffer, CNBuffSize, + compressedBuffer, cSize, + dictBuffer, dictSize), + if (r != CNBuffSize) goto _output_error); + ZSTD_freeDCtx(dctx); + } DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3i : compress with static CDict : ", testNb++); @@ -944,7 +1119,7 @@ static int basicUnitTests(U32 seed, double compressibility) } } free(cdictBuffer); } } - DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++); { ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ }; @@ -955,7 +1130,7 @@ static int basicUnitTests(U32 seed, double compressibility) ZSTD_freeCDict(cdict); if (ZSTD_isError(cSize)) goto _output_error; } - DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); DISPLAYLEVEL(3, "test%3i : try retrieving contentSize from frame : ", testNb++); { U64 const contentSize = ZSTD_getFrameContentSize(compressedBuffer, cSize); @@ -964,11 +1139,14 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "OK (unknown)\n"); DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++); - CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, - decodedBuffer, CNBuffSize, - compressedBuffer, cSize, - dictBuffer, dictSize), - if (r != CNBuffSize) goto _output_error); + { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); + CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, + decodedBuffer, CNBuffSize, + compressedBuffer, cSize, + dictBuffer, dictSize), + if (r != CNBuffSize) goto _output_error); + ZSTD_freeDCtx(dctx); + } DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb++); @@ -979,22 +1157,29 @@ static int basicUnitTests(U32 seed, double compressibility) dictBuffer, dictSize, p); if (ZSTD_isError(cSize)) goto _output_error; } - DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++); - CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, - decodedBuffer, CNBuffSize, - compressedBuffer, cSize, - dictBuffer, dictSize), - if (r != CNBuffSize) goto _output_error); + { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); + CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, + decodedBuffer, CNBuffSize, + compressedBuffer, cSize, + dictBuffer, dictSize), + if (r != CNBuffSize) goto _output_error); + ZSTD_freeDCtx(dctx); + } DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3i : dictionary containing only header should return error : ", testNb++); - { - const size_t ret = ZSTD_decompress_usingDict( - dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, - "\x37\xa4\x30\xec\x11\x22\x33\x44", 8); - if (ZSTD_getErrorCode(ret) != ZSTD_error_dictionary_corrupted) goto _output_error; + { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + assert(dctx != NULL); + { const size_t ret = ZSTD_decompress_usingDict( + dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, + "\x37\xa4\x30\xec\x11\x22\x33\x44", 8); + if (ZSTD_getErrorCode(ret) != ZSTD_error_dictionary_corrupted) + goto _output_error; + } + ZSTD_freeDCtx(dctx); } DISPLAYLEVEL(3, "OK \n"); @@ -1063,10 +1248,12 @@ static int basicUnitTests(U32 seed, double compressibility) */ { size_t dSize; BYTE data[1024]; + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); ZSTD_compressionParameters const cParams = ZSTD_getCParams(19, CNBuffSize, dictSize); ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); + assert(dctx != NULL); assert(cdict != NULL); memset(data, 'x', sizeof(data)); cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, data, sizeof(data), cdict); @@ -1075,6 +1262,7 @@ static int basicUnitTests(U32 seed, double compressibility) dSize = ZSTD_decompress_usingDict(dctx, decodedBuffer, sizeof(data), compressedBuffer, cSize, dictBuffer, dictSize); if (ZSTD_isError(dSize)) { DISPLAYLEVEL(5, "Decompression error %s : ", ZSTD_getErrorName(dSize)); goto _output_error; } if (memcmp(data, decodedBuffer, sizeof(data))) { DISPLAYLEVEL(5, "Data corruption : "); goto _output_error; } + ZSTD_freeDCtx(dctx); } DISPLAYLEVEL(3, "OK \n"); @@ -1110,12 +1298,12 @@ static int basicUnitTests(U32 seed, double compressibility) CNBuffer, samplesSizes, nbSamples, params); if (ZDICT_isError(dictSize)) goto _output_error; - DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (U32)dictSize); + DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++); dictID = ZDICT_getDictID(dictBuffer, dictSize); if (dictID==0) goto _output_error; - DISPLAYLEVEL(3, "OK : %u \n", dictID); + DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID); DISPLAYLEVEL(3, "test%3i : ZDICT_optimizeTrainFromBuffer_cover : ", testNb++); memset(¶ms, 0, sizeof(params)); @@ -1124,12 +1312,12 @@ static int basicUnitTests(U32 seed, double compressibility) CNBuffer, samplesSizes, nbSamples / 4, ¶ms); if (ZDICT_isError(optDictSize)) goto _output_error; - DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (U32)optDictSize); + DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)optDictSize); DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++); dictID = ZDICT_getDictID(dictBuffer, optDictSize); if (dictID==0) goto _output_error; - DISPLAYLEVEL(3, "OK : %u \n", dictID); + DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID); ZSTD_freeCCtx(cctx); free(dictBuffer); @@ -1180,13 +1368,13 @@ static int basicUnitTests(U32 seed, double compressibility) params); if (ZSTD_isError(cSize_1pass)) goto _output_error; - CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, (unsigned)compressionLevel) ); - { ZSTD_inBuffer in = { CNBuffer, srcSize, 0 }; - ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 }; - size_t const compressionResult = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end); - DISPLAYLEVEL(5, "simple=%zu vs %zu=advanced : ", cSize_1pass, out.pos); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) ); + { size_t const compressionResult = ZSTD_compress2(cctx, + compressedBuffer, compressedBufferSize, + CNBuffer, srcSize); + DISPLAYLEVEL(5, "simple=%zu vs %zu=advanced : ", cSize_1pass, compressionResult); if (ZSTD_isError(compressionResult)) goto _output_error; - if (out.pos != cSize_1pass) goto _output_error; + if (compressionResult != cSize_1pass) goto _output_error; } } ZSTD_freeCCtx(cctx); } @@ -1199,54 +1387,86 @@ static int basicUnitTests(U32 seed, double compressibility) { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++); assert(cctx != NULL); - CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, 2) ); - CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_enableLongDistanceMatching, 1) ); - CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_windowLog, 18) ); - { ZSTD_inBuffer in = { CNBuffer, inputSize, 0 }; - ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 }; - size_t const result = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end); - if (result != 0) goto _output_error; - if (in.pos != in.size) goto _output_error; - cSize = out.pos; - xxh64 = XXH64(out.dst, out.pos, 0); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) ); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1) ); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) ); + { size_t const compressedSize = ZSTD_compress2(cctx, + compressedBuffer, ZSTD_compressBound(inputSize), + CNBuffer, inputSize); + CHECK(compressedSize); + cSize = compressedSize; + xxh64 = XXH64(compressedBuffer, compressedSize, 0); } - DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (U32)inputSize, (U32)cSize); + DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize); ZSTD_freeCCtx(cctx); } { ZSTD_CCtx* cctx = ZSTD_createCCtx(); DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++); - CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_windowLog, 18) ); - CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_enableLongDistanceMatching, 1) ); - CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, 2) ); - { ZSTD_inBuffer in = { CNBuffer, inputSize, 0 }; - ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 }; - size_t const result = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end); - if (result != 0) goto _output_error; - if (in.pos != in.size) goto _output_error; - if (out.pos != cSize) goto _output_error; /* must result in same compressed result, hence same size */ - if (XXH64(out.dst, out.pos, 0) != xxh64) goto _output_error; /* must result in exactly same content, hence same hash */ - DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (U32)inputSize, (U32)out.pos); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) ); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1) ); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) ); + { size_t const result = ZSTD_compress2(cctx, + compressedBuffer, ZSTD_compressBound(inputSize), + CNBuffer, inputSize); + CHECK(result); + if (result != cSize) goto _output_error; /* must result in same compressed result, hence same size */ + if (XXH64(compressedBuffer, result, 0) != xxh64) goto _output_error; /* must result in exactly same content, hence same hash */ + DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)result); } ZSTD_freeCCtx(cctx); } } + /* advanced parameters for decompression */ + { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + assert(dctx != NULL); + + DISPLAYLEVEL(3, "test%3i : get dParameter bounds ", testNb++); + { ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax); + CHECK(bounds.error); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : wrong dParameter : ", testNb++); + { size_t const sr = ZSTD_DCtx_setParameter(dctx, (ZSTD_dParameter)999999, 0); + if (!ZSTD_isError(sr)) goto _output_error; + } + { ZSTD_bounds const bounds = ZSTD_dParam_getBounds((ZSTD_dParameter)999998); + if (!ZSTD_isError(bounds.error)) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : out of bound dParameter : ", testNb++); + { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 9999); + if (!ZSTD_isError(sr)) goto _output_error; + } + { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (ZSTD_format_e)888); + if (!ZSTD_isError(sr)) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + + ZSTD_freeDCtx(dctx); + } + + /* custom formats tests */ { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); size_t const inputSize = CNBuffSize / 2; /* won't cause pb with small dict size */ + assert(dctx != NULL); assert(cctx != NULL); /* basic block compression */ DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb++); - CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_format, ZSTD_f_zstd1_magicless) ); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) ); { ZSTD_inBuffer in = { CNBuffer, inputSize, 0 }; ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 }; - size_t const result = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end); + size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end); if (result != 0) goto _output_error; if (in.pos != in.size) goto _output_error; cSize = out.pos; } - DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (U32)inputSize, (U32)cSize); + DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize); DISPLAYLEVEL(3, "test%3i : decompress normally (should fail) : ", testNb++); { size_t const decodeResult = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); @@ -1255,29 +1475,38 @@ static int basicUnitTests(U32 seed, double compressibility) } DISPLAYLEVEL(3, "test%3i : decompress of magic-less frame : ", testNb++); - ZSTD_DCtx_reset(dctx); - CHECK( ZSTD_DCtx_setFormat(dctx, ZSTD_f_zstd1_magicless) ); + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); + CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) ); { ZSTD_frameHeader zfh; size_t const zfhrt = ZSTD_getFrameHeader_advanced(&zfh, compressedBuffer, cSize, ZSTD_f_zstd1_magicless); if (zfhrt != 0) goto _output_error; } + /* one shot */ + { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); + if (result != inputSize) goto _output_error; + DISPLAYLEVEL(3, "one-shot OK, "); + } + /* streaming */ { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 }; ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 }; - size_t const result = ZSTD_decompress_generic(dctx, &out, &in); + size_t const result = ZSTD_decompressStream(dctx, &out, &in); if (result != 0) goto _output_error; if (in.pos != in.size) goto _output_error; if (out.pos != inputSize) goto _output_error; - DISPLAYLEVEL(3, "OK : regenerated %u bytes \n", (U32)out.pos); + DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos); } ZSTD_freeCCtx(cctx); + ZSTD_freeDCtx(dctx); } /* block API tests */ { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); static const size_t dictSize = 65 KB; static const size_t blockSize = 100 KB; /* won't cause pb with small dict size */ size_t cSize2; + assert(cctx != NULL); assert(dctx != NULL); /* basic block compression */ DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++); @@ -1295,13 +1524,14 @@ static int basicUnitTests(U32 seed, double compressibility) /* very long stream of block compression */ DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb++); - CHECK( ZSTD_compressBegin(cctx, -99) ); /* we just want to quickly overflow internal U32 index */ + CHECK( ZSTD_compressBegin(cctx, -199) ); /* we just want to quickly overflow internal U32 index */ CHECK( ZSTD_getBlockSize(cctx) >= blockSize); { U64 const toCompress = 5000000000ULL; /* > 4 GB */ U64 compressed = 0; while (compressed < toCompress) { size_t const blockCSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize); - if (ZSTD_isError(cSize)) goto _output_error; + assert(blockCSize != 0); + if (ZSTD_isError(blockCSize)) goto _output_error; compressed += blockCSize; } } @@ -1339,8 +1569,8 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "OK \n"); ZSTD_freeCCtx(cctx); + ZSTD_freeDCtx(dctx); } - ZSTD_freeDCtx(dctx); /* long rle test */ { size_t sampleSize = 0; @@ -1363,7 +1593,7 @@ static int basicUnitTests(U32 seed, double compressibility) memset(CNBuffer, 0, ZEROESLENGTH); { CHECK_V(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) ); cSize = r; } - DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/ZEROESLENGTH*100); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/ZEROESLENGTH*100); DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH); { CHECK_V(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) ); @@ -1417,7 +1647,7 @@ static int basicUnitTests(U32 seed, double compressibility) { CHECK_V(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH), CNBuffer, _3BYTESTESTLENGTH, 19) ); cSize = r; } - DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/_3BYTESTESTLENGTH*100); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/_3BYTESTESTLENGTH*100); DISPLAYLEVEL(3, "test%3i : decompress lots 3-bytes sequence : ", testNb++); { CHECK_V(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) ); @@ -1586,7 +1816,7 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog) if (cond) { \ DISPLAY("Error => "); \ DISPLAY(__VA_ARGS__); \ - DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); \ + DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \ goto _output_error; \ } } @@ -1596,12 +1826,12 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog) if (ZSTD_isError(err)) { \ DISPLAY("Error => %s : %s ", \ #f, ZSTD_getErrorName(err)); \ - DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); \ + DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \ goto _output_error; \ } } -static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxDurationS, double compressibility, int bigTests) +static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, U32 const maxDurationS, double compressibility, int bigTests) { static const U32 maxSrcLog = 23; static const U32 maxSampleLog = 22; @@ -1616,7 +1846,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD ZSTD_CCtx* const ctx = ZSTD_createCCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); U32 result = 0; - U32 testNb = 0; + unsigned testNb = 0; U32 coreSeed = seed; UTIL_time_t const startClock = UTIL_getTime(); U64 const maxClockSpan = maxDurationS * SEC_TO_MICRO; @@ -1698,15 +1928,17 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed : %s", ZSTD_getErrorName(cSize)); /* compression failure test : too small dest buffer */ - if (cSize > 3) { - const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ + assert(cSize > 3); + { const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; const size_t tooSmallSize = cSize - missing; - const U32 endMark = 0x4DC2B1A9; - memcpy(dstBuffer+tooSmallSize, &endMark, 4); + const unsigned endMark = 0x4DC2B1A9; + memcpy(dstBuffer+tooSmallSize, &endMark, sizeof(endMark)); + DISPLAYLEVEL(5, "fuzzer t%u: compress into too small buffer of size %u (missing %u bytes) \n", + testNb, (unsigned)tooSmallSize, (unsigned)missing); { size_t const errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel); - CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize); } - { U32 endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, 4); - CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); } + CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (unsigned)tooSmallSize, (unsigned)cSize); } + { unsigned endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, sizeof(endCheck)); + CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow (check.%08X != %08X.mark)", endCheck, endMark); } } } /* frame header decompression test */ @@ -1724,9 +1956,9 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD DISPLAYLEVEL(5, "fuzzer t%u: simple decompression test \n", testNb); { size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1; size_t const dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize); - CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize); + CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (unsigned)sampleSize, (unsigned)cSize); { U64 const crcDest = XXH64(dstBuffer, sampleSize, 0); - CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(sampleBuffer, dstBuffer, sampleSize), (U32)sampleSize); + CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (unsigned)findDiff(sampleBuffer, dstBuffer, sampleSize), (unsigned)sampleSize); } } free(sampleBuffer); /* no longer useful after this point */ @@ -1751,7 +1983,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD static const BYTE token = 0xA9; dstBuffer[tooSmallSize] = token; { size_t const errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize); - CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (U32)errorCode, (U32)tooSmallSize); } + CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (unsigned)errorCode, (unsigned)tooSmallSize); } CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow"); } @@ -1786,7 +2018,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD { size_t const decompressResult = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize); /* result *may* be an unlikely success, but even then, it must strictly respect dst buffer boundaries */ CHECK((!ZSTD_isError(decompressResult)) && (decompressResult>sampleSize), - "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (U32)decompressResult, (U32)sampleSize); + "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (unsigned)decompressResult, (unsigned)sampleSize); } { U32 endCheck; memcpy(&endCheck, dstBuffer+sampleSize, 4); CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow"); @@ -1807,7 +2039,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize)); DISPLAYLEVEL(6, "fuzzer t%u: Compressing up to <=%u bytes at level %i with dictionary size %u \n", - testNb, (U32)maxTestSize, cLevel, (U32)dictSize); + testNb, (unsigned)maxTestSize, cLevel, (unsigned)dictSize); if (FUZ_rand(&lseed) & 0xF) { CHECK_Z ( ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel) ); @@ -1853,13 +2085,13 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming decompression test \n", testNb); /* ensure memory requirement is good enough (should always be true) */ { ZSTD_frameHeader zfh; - CHECK( ZSTD_getFrameHeader(&zfh, cBuffer, ZSTD_frameHeaderSize_max), + CHECK( ZSTD_getFrameHeader(&zfh, cBuffer, ZSTD_FRAMEHEADERSIZE_MAX), "ZSTD_getFrameHeader(): error retrieving frame information"); { size_t const roundBuffSize = ZSTD_decodingBufferSize_min(zfh.windowSize, zfh.frameContentSize); CHECK_Z(roundBuffSize); CHECK((roundBuffSize > totalTestSize) && (zfh.frameContentSize!=ZSTD_CONTENTSIZE_UNKNOWN), "ZSTD_decodingBufferSize_min() requires more memory (%u) than necessary (%u)", - (U32)roundBuffSize, (U32)totalTestSize ); + (unsigned)roundBuffSize, (unsigned)totalTestSize ); } } if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */ CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, dict, dictSize) ); @@ -1877,7 +2109,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD CHECK (totalCSize != cSize, "compressed data should be fully read") { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); CHECK(crcOrig != crcDest, "streaming decompressed data corrupted (pos %u / %u)", - (U32)findDiff(mirrorBuffer, dstBuffer, totalTestSize), (U32)totalTestSize); + (unsigned)findDiff(mirrorBuffer, dstBuffer, totalTestSize), (unsigned)totalTestSize); } } /* for ( ; (testNb <= nbTests) */ DISPLAY("\r%u fuzzer tests completed \n", testNb-1); @@ -1911,10 +2143,10 @@ static int FUZ_usage(const char* programName) DISPLAY( " %s [args]\n", programName); DISPLAY( "\n"); DISPLAY( "Arguments :\n"); - DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault); + DISPLAY( " -i# : Nb of tests (default:%i) \n", nbTestsDefault); DISPLAY( " -s# : Select seed (default:prompt user)\n"); DISPLAY( " -t# : Select starting test number (default:0)\n"); - DISPLAY( " -P# : Select compressibility in %% (default:%u%%)\n", FUZ_compressibility_default); + DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_compressibility_default); DISPLAY( " -v : verbose\n"); DISPLAY( " -p : pause at the end\n"); DISPLAY( " -h : display help and exit\n"); @@ -1961,7 +2193,7 @@ int main(int argc, const char** argv) int argNb; int nbTests = nbTestsDefault; int testNb = 0; - U32 proba = FUZ_compressibility_default; + int proba = FUZ_compressibility_default; int result = 0; U32 mainPause = 0; U32 maxDuration = 0; @@ -2048,8 +2280,8 @@ int main(int argc, const char** argv) seed = h % 10000; } - DISPLAY("Seed = %u\n", seed); - if (proba!=FUZ_compressibility_default) DISPLAY("Compressibility : %u%%\n", proba); + DISPLAY("Seed = %u\n", (unsigned)seed); + if (proba!=FUZ_compressibility_default) DISPLAY("Compressibility : %i%%\n", proba); if (memTestsOnly) { g_displayLevel = MAX(3, g_displayLevel); diff --git a/tests/libzstd_partial_builds.sh b/tests/libzstd_partial_builds.sh index 34d8ea55231e..b1c1e3b1a7e3 100755 --- a/tests/libzstd_partial_builds.sh +++ b/tests/libzstd_partial_builds.sh @@ -1,36 +1,89 @@ #!/bin/sh -e +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +ECHO=echo +RM="rm -f" +GREP="grep" +INTOVOID="/dev/null" + die() { $ECHO "$@" 1>&2 exit 1 } -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +isPresent() { + $GREP $@ tmplog || die "$@" "should be present" +} -INTOVOID="/dev/null" -case "$OS" in - Windows*) - INTOVOID="NUL" - ;; -esac +mustBeAbsent() { + $GREP $@ tmplog && die "$@ should not be there !!" + $ECHO "$@ correctly not present" # for some reason, this $ECHO must exist, otherwise mustBeAbsent() always fails (??) +} -ZSTD_LIB_COMPRESSION=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID -nm $DIR/../lib/libzstd.a | grep ".*\.o:" > tmplog -! grep -q "zstd_compress" tmplog && grep -q "zstd_decompress" tmplog && ! grep -q "dict" tmplog && grep -q "zstd_v" tmplog && ! grep -q "zbuff" tmplog && make clean && rm -f tmplog || die "Compression macro failed" +# default compilation : all features enabled +make clean > /dev/null +$ECHO "testing default library compilation" +CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID +nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog +isPresent "zstd_compress.o" +isPresent "zstd_decompress.o" +isPresent "zdict.o" +isPresent "zstd_v07.o" +isPresent "zbuff_compress.o" +$RM $DIR/../lib/libzstd.a tmplog +# compression disabled => also disable zdict and zbuff +$ECHO "testing with compression disabled" +ZSTD_LIB_COMPRESSION=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID +nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog +mustBeAbsent "zstd_compress.o" +isPresent "zstd_decompress.o" +mustBeAbsent "zdict.o" +isPresent "zstd_v07.o" +mustBeAbsent "zbuff_compress.o" +$RM $DIR/../lib/libzstd.a tmplog +# decompression disabled => also disable legacy and zbuff +$ECHO "testing with decompression disabled" ZSTD_LIB_DECOMPRESSION=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID -nm $DIR/../lib/libzstd.a | grep ".*\.o:" > tmplog -grep -q "zstd_compress" tmplog && ! grep -q "zstd_decompress" tmplog && grep -q "dict" tmplog && ! grep -q "zstd_v" tmplog && ! grep -q "zbuff" tmplog && make clean && rm -f tmplog || die "Decompression macro failed" +nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog +isPresent "zstd_compress.o" +mustBeAbsent "zstd_decompress.o" +isPresent "zdict.o" +mustBeAbsent "zstd_v07.o" +mustBeAbsent "zbuff_compress.o" +$RM $DIR/../lib/libzstd.a tmplog +# deprecated function disabled => only remove zbuff +$ECHO "testing with deprecated functions disabled" ZSTD_LIB_DEPRECATED=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID -nm $DIR/../lib/libzstd.a | grep ".*\.o:" > tmplog -grep -q "zstd_compress" tmplog && grep -q "zstd_decompress" tmplog && grep -q "dict" tmplog && grep -q "zstd_v" tmplog && ! grep -q "zbuff" tmplog && make clean && rm -f tmplog || die "Deprecated macro failed" +nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog +isPresent "zstd_compress.o" +isPresent "zstd_decompress.o" +isPresent "zdict.o" +isPresent "zstd_v07.o" +mustBeAbsent "zbuff_compress.o" +$RM $DIR/../lib/libzstd.a tmplog +# dictionary builder disabled => only remove zdict +$ECHO "testing with dictionary builder disabled" ZSTD_LIB_DICTBUILDER=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID -nm $DIR/../lib/libzstd.a | grep ".*\.o:" > tmplog -grep -q "zstd_compress" tmplog && grep -q "zstd_decompress" tmplog && ! grep -q "dict" tmplog && grep -q "zstd_v" tmplog && grep -q "zbuff" tmplog && make clean && rm -f tmplog || die "Dictbuilder macro failed" +nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog +isPresent "zstd_compress.o" +isPresent "zstd_decompress.o" +mustBeAbsent "zdict.o" +isPresent "zstd_v07.o" +isPresent "zbuff_compress.o" +$RM $DIR/../lib/libzstd.a tmplog +# both decompression and dictionary builder disabled => only compression remains +$ECHO "testing with both decompression and dictionary builder disabled (only compression remains)" ZSTD_LIB_DECOMPRESSION=0 ZSTD_LIB_DICTBUILDER=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID -nm $DIR/../lib/libzstd.a | grep ".*\.o:" > tmplog -grep -q "zstd_compress" tmplog && ! grep -q "zstd_decompress" tmplog && ! grep -q "dict" tmplog && ! grep -q "zstd_v" tmplog && ! grep -q "zbuff" tmplog && make clean && rm -f tmplog || die "Multi-macro failed"
\ No newline at end of file +nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog +isPresent "zstd_compress.o" +mustBeAbsent "zstd_decompress.o" +mustBeAbsent "zdict.o" +mustBeAbsent "zstd_v07.o" +mustBeAbsent "zbuff_compress.o" +$RM $DIR/../lib/libzstd.a tmplog diff --git a/tests/longmatch.c b/tests/longmatch.c index 1271e9ab1039..b673baa60140 100644 --- a/tests/longmatch.c +++ b/tests/longmatch.c @@ -50,7 +50,7 @@ int main(int argc, const char** argv) params.cParams.chainLog = 13; params.cParams.hashLog = 14; params.cParams.searchLog = 1; - params.cParams.searchLength = 7; + params.cParams.minMatch = 7; params.cParams.targetLength = 16; params.cParams.strategy = ZSTD_fast; windowLog = params.cParams.windowLog; diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 7a4be854a46b..415551b9578d 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -12,7 +12,7 @@ /*-************************************ * Dependencies **************************************/ -#include "util.h" /* Compiler options, UTIL_GetFileSize */ +#include "util.h" /* Ensure platform.h is compiled first; also : compiler options, UTIL_GetFileSize */ #include <stdlib.h> /* malloc */ #include <stdio.h> /* fprintf, fopen, ftello64 */ #include <string.h> /* strcmp */ @@ -24,8 +24,8 @@ #include "zstd.h" #include "datagen.h" #include "xxhash.h" -#include "util.h" -#include "bench.h" +#include "benchfn.h" +#include "benchzstd.h" #include "zstd_errors.h" #include "zstd_internal.h" /* should not be needed */ @@ -74,20 +74,21 @@ static const int g_maxNbVariations = 64; #define CLOG_RANGE (ZSTD_CHAINLOG_MAX - ZSTD_CHAINLOG_MIN + 1) #define HLOG_RANGE (ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1) #define SLOG_RANGE (ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1) -#define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1) -#define TLEN_RANGE 17 -#define STRT_RANGE (ZSTD_btultra - ZSTD_fast + 1) -#define FADT_RANGE 3 +#define MML_RANGE (ZSTD_MINMATCH_MAX - ZSTD_MINMATCH_MIN + 1) +#define TLEN_RANGE 17 +#define STRT_RANGE (ZSTD_STRATEGY_MAX - ZSTD_STRATEGY_MIN + 1) +#define FADT_RANGE 3 -#define CHECKTIME(r) { if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); return r; } } -#define CHECKTIMEGT(ret, val, _gototag) {if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); ret = val; goto _gototag; } } +#define CHECKTIME(r) { if(BMK_timeSpan_s(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); return r; } } +#define CHECKTIMEGT(ret, val, _gototag) { if(BMK_timeSpan_s(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); ret = val; goto _gototag; } } #define PARAM_UNSET ((U32)-2) /* can't be -1 b/c fadt uses -1 */ -static const char* g_stratName[ZSTD_btultra+1] = { +static const char* g_stratName[ZSTD_STRATEGY_MAX+1] = { "(none) ", "ZSTD_fast ", "ZSTD_dfast ", "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", - "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; + "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra ", + "ZSTD_btultra2"}; static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 }; @@ -102,7 +103,7 @@ typedef enum { clog_ind = 1, hlog_ind = 2, slog_ind = 3, - slen_ind = 4, + mml_ind = 4, tlen_ind = 5, strt_ind = 6, fadt_ind = 7, /* forceAttachDict */ @@ -113,56 +114,66 @@ typedef struct { U32 vals[NUM_PARAMS]; } paramValues_t; -/* maximum value of parameters */ +/* minimum value of parameters */ static const U32 mintable[NUM_PARAMS] = - { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN, ZSTD_fast, FADT_MIN }; + { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_MINMATCH_MIN, ZSTD_TARGETLENGTH_MIN, ZSTD_STRATEGY_MIN, FADT_MIN }; -/* minimum value of parameters */ +/* maximum value of parameters */ static const U32 maxtable[NUM_PARAMS] = - { ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_SEARCHLENGTH_MAX, ZSTD_TARGETLENGTH_MAX, ZSTD_btultra, FADT_MAX }; + { ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_MINMATCH_MAX, ZSTD_TARGETLENGTH_MAX, ZSTD_STRATEGY_MAX, FADT_MAX }; /* # of values parameters can take on */ static const U32 rangetable[NUM_PARAMS] = - { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE, FADT_RANGE }; + { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, MML_RANGE, TLEN_RANGE, STRT_RANGE, FADT_RANGE }; /* ZSTD_cctxSetParameter() index to set */ static const ZSTD_cParameter cctxSetParamTable[NUM_PARAMS] = - { ZSTD_p_windowLog, ZSTD_p_chainLog, ZSTD_p_hashLog, ZSTD_p_searchLog, ZSTD_p_minMatch, ZSTD_p_targetLength, ZSTD_p_compressionStrategy, ZSTD_p_forceAttachDict }; + { ZSTD_c_windowLog, ZSTD_c_chainLog, ZSTD_c_hashLog, ZSTD_c_searchLog, ZSTD_c_minMatch, ZSTD_c_targetLength, ZSTD_c_strategy, ZSTD_c_forceAttachDict }; /* names of parameters */ static const char* g_paramNames[NUM_PARAMS] = - { "windowLog", "chainLog", "hashLog","searchLog", "searchLength", "targetLength", "strategy", "forceAttachDict" }; + { "windowLog", "chainLog", "hashLog","searchLog", "minMatch", "targetLength", "strategy", "forceAttachDict" }; /* shortened names of parameters */ static const char* g_shortParamNames[NUM_PARAMS] = - { "wlog", "clog", "hlog","slog", "slen", "tlen", "strt", "fadt" }; + { "wlog", "clog", "hlog", "slog", "mml", "tlen", "strat", "fadt" }; /* maps value from { 0 to rangetable[param] - 1 } to valid paramvalues */ -static U32 rangeMap(varInds_t param, int ind) { +static U32 rangeMap(varInds_t param, int ind) +{ ind = MAX(MIN(ind, (int)rangetable[param] - 1), 0); switch(param) { - case tlen_ind: - return tlen_table[ind]; - case fadt_ind: /* 0, 1, 2 -> -1, 0, 1 */ - return ind - 1; case wlog_ind: /* using default: triggers -Wswitch-enum */ case clog_ind: case hlog_ind: case slog_ind: - case slen_ind: + case mml_ind: case strt_ind: return mintable[param] + ind; + case tlen_ind: + return tlen_table[ind]; + case fadt_ind: /* 0, 1, 2 -> -1, 0, 1 */ + return ind - 1; case NUM_PARAMS: - DISPLAY("Error, not a valid param\n "); - return (U32)-1; + default:; } - return 0; /* should never happen, stop compiler warnings */ + DISPLAY("Error, not a valid param\n "); + assert(0); + return (U32)-1; } /* inverse of rangeMap */ -static int invRangeMap(varInds_t param, U32 value) { +static int invRangeMap(varInds_t param, U32 value) +{ value = MIN(MAX(mintable[param], value), maxtable[param]); switch(param) { + case wlog_ind: + case clog_ind: + case hlog_ind: + case slog_ind: + case mml_ind: + case strt_ind: + return value - mintable[param]; case tlen_ind: /* bin search */ { int lo = 0; @@ -181,33 +192,49 @@ static int invRangeMap(varInds_t param, U32 value) { } case fadt_ind: return (int)value + 1; - case wlog_ind: - case clog_ind: - case hlog_ind: - case slog_ind: - case slen_ind: - case strt_ind: - return value - mintable[param]; case NUM_PARAMS: - DISPLAY("Error, not a valid param\n "); - return -2; + default:; } - return 0; /* should never happen, stop compiler warnings */ + DISPLAY("Error, not a valid param\n "); + assert(0); + return -2; } /* display of params */ -static void displayParamVal(FILE* f, varInds_t param, U32 value, int width) { +static void displayParamVal(FILE* f, varInds_t param, unsigned value, int width) +{ switch(param) { - case fadt_ind: if(width) { fprintf(f, "%*d", width, (int)value); } else { fprintf(f, "%d", (int)value); } break; - case strt_ind: if(width) { fprintf(f, "%*s", width, g_stratName[value]); } else { fprintf(f, "%s", g_stratName[value]); } break; case wlog_ind: case clog_ind: case hlog_ind: case slog_ind: - case slen_ind: - case tlen_ind: if(width) { fprintf(f, "%*u", width, value); } else { fprintf(f, "%u", value); } break; + case mml_ind: + case tlen_ind: + if(width) { + fprintf(f, "%*u", width, value); + } else { + fprintf(f, "%u", value); + } + break; + case strt_ind: + if(width) { + fprintf(f, "%*s", width, g_stratName[value]); + } else { + fprintf(f, "%s", g_stratName[value]); + } + break; + case fadt_ind: /* force attach dict */ + if(width) { + fprintf(f, "%*d", width, (int)value); + } else { + fprintf(f, "%d", (int)value); + } + break; case NUM_PARAMS: - DISPLAY("Error, not a valid param\n "); break; + default: + DISPLAY("Error, not a valid param\n "); + assert(0); + break; } } @@ -216,8 +243,6 @@ static void displayParamVal(FILE* f, varInds_t param, U32 value, int width) { * Benchmark Parameters/Global Variables **************************************/ -typedef BYTE U8; - /* General Utility */ static U32 g_timeLimit_s = 99999; /* about 27 hours */ static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ @@ -226,7 +251,7 @@ static U32 g_rand = 1; /* Display */ static int g_displayLevel = 3; -static BYTE g_silenceParams[NUM_PARAMS]; +static BYTE g_silenceParams[NUM_PARAMS]; /* can selectively silence some params when displaying them */ /* Mode Selection */ static U32 g_singleRun = 0; @@ -297,50 +322,54 @@ static paramValues_t sanitizeParams(paramValues_t params) params.vals[clog_ind] = 0, params.vals[slog_ind] = 0; if (params.vals[strt_ind] == ZSTD_dfast) params.vals[slog_ind] = 0; - if (params.vals[strt_ind] != ZSTD_btopt && params.vals[strt_ind] != ZSTD_btultra && params.vals[strt_ind] != ZSTD_fast) + if ( (params.vals[strt_ind] < ZSTD_btopt) && (params.vals[strt_ind] != ZSTD_fast) ) params.vals[tlen_ind] = 0; return params; } -static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) { +static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) +{ ZSTD_compressionParameters c; memset(&c, 0, sizeof(ZSTD_compressionParameters)); c.windowLog = p.vals[wlog_ind]; c.chainLog = p.vals[clog_ind]; c.hashLog = p.vals[hlog_ind]; c.searchLog = p.vals[slog_ind]; - c.searchLength = p.vals[slen_ind]; + c.minMatch = p.vals[mml_ind]; c.targetLength = p.vals[tlen_ind]; c.strategy = p.vals[strt_ind]; /* no forceAttachDict */ return c; } -static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) { +static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) +{ paramValues_t p; varInds_t i; p.vals[wlog_ind] = c.windowLog; p.vals[clog_ind] = c.chainLog; p.vals[hlog_ind] = c.hashLog; p.vals[slog_ind] = c.searchLog; - p.vals[slen_ind] = c.searchLength; + p.vals[mml_ind] = c.minMatch; p.vals[tlen_ind] = c.targetLength; p.vals[strt_ind] = c.strategy; /* set all other params to their minimum value */ - for(i = strt_ind + 1; i < NUM_PARAMS; i++) { + for (i = strt_ind + 1; i < NUM_PARAMS; i++) { p.vals[i] = mintable[i]; } return p; } /* equivalent of ZSTD_adjustCParams for paramValues_t */ -static paramValues_t adjustParams(paramValues_t p, const size_t maxBlockSize, const size_t dictSize) { +static paramValues_t +adjustParams(paramValues_t p, const size_t maxBlockSize, const size_t dictSize) +{ paramValues_t ot = p; varInds_t i; p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize)); - if(!dictSize) { p.vals[fadt_ind] = 0; } + if (!dictSize) { p.vals[fadt_ind] = 0; } /* retain value of all other parameters */ for(i = strt_ind + 1; i < NUM_PARAMS; i++) { p.vals[i] = ot.vals[i]; @@ -367,7 +396,10 @@ static size_t BMK_findMaxMem(U64 requiredMem) } /* accuracy in seconds only, span can be multiple years */ -static U32 BMK_timeSpan(const UTIL_time_t tStart) { return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); } +static U32 BMK_timeSpan_s(const UTIL_time_t tStart) +{ + return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); +} static U32 FUZ_rotl32(U32 x, U32 r) { @@ -386,32 +418,38 @@ static U32 FUZ_rand(U32* src) return rand32 >> 5; } -/* allows zeros */ -#define CLAMPCHECK(val,min,max) { \ +#define BOUNDCHECK(val,min,max) { \ if (((val)<(min)) | ((val)>(max))) { \ DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); \ return 0; \ } } -static int paramValid(const paramValues_t paramTarget) { +static int paramValid(const paramValues_t paramTarget) +{ U32 i; for(i = 0; i < NUM_PARAMS; i++) { - CLAMPCHECK(paramTarget.vals[i], mintable[i], maxtable[i]); + BOUNDCHECK(paramTarget.vals[i], mintable[i], maxtable[i]); } return 1; } -static paramValues_t cParamUnsetMin(paramValues_t paramTarget) { - varInds_t i; - for(i = 0; i < NUM_PARAMS; i++) { - if(paramTarget.vals[i] == PARAM_UNSET) { - paramTarget.vals[i] = mintable[i]; +/* cParamUnsetMin() : + * if any parameter in paramTarget is not yet set, + * it will receive its corresponding minimal value. + * This function never fails */ +static paramValues_t cParamUnsetMin(paramValues_t paramTarget) +{ + varInds_t vi; + for (vi = 0; vi < NUM_PARAMS; vi++) { + if (paramTarget.vals[vi] == PARAM_UNSET) { + paramTarget.vals[vi] = mintable[vi]; } } return paramTarget; } -static paramValues_t emptyParams(void) { +static paramValues_t emptyParams(void) +{ U32 i; paramValues_t p; for(i = 0; i < NUM_PARAMS; i++) { @@ -420,7 +458,8 @@ static paramValues_t emptyParams(void) { return p; } -static winnerInfo_t initWinnerInfo(const paramValues_t p) { +static winnerInfo_t initWinnerInfo(const paramValues_t p) +{ winnerInfo_t w1; w1.result.cSpeed = 0.; w1.result.dSpeed = 0.; @@ -430,7 +469,9 @@ static winnerInfo_t initWinnerInfo(const paramValues_t p) { return w1; } -static paramValues_t overwriteParams(paramValues_t base, const paramValues_t mask) { +static paramValues_t +overwriteParams(paramValues_t base, const paramValues_t mask) +{ U32 i; for(i = 0; i < NUM_PARAMS; i++) { if(mask.vals[i] != PARAM_UNSET) { @@ -440,12 +481,16 @@ static paramValues_t overwriteParams(paramValues_t base, const paramValues_t mas return base; } -static void paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues_t* ptr) { - ptr->vals[paramIndex] = rangeMap(paramIndex, invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt); +static void +paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues_t* ptr) +{ + ptr->vals[paramIndex] = rangeMap(paramIndex, + invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt); } /* varies ptr by nbChanges respecting varyParams*/ -static void paramVariation(paramValues_t* ptr, memoTable_t* mtAll, const U32 nbChanges) +static void +paramVariation(paramValues_t* ptr, memoTable_t* mtAll, const U32 nbChanges) { paramValues_t p; U32 validated = 0; @@ -473,8 +518,9 @@ static paramValues_t randomParams(void) static U64 g_clockGranularity = 100000000ULL; -static void findClockGranularity(void) { - UTIL_time_t clockStart = UTIL_getTime(); +static void init_clockGranularity(void) +{ + UTIL_time_t const clockStart = UTIL_getTime(); U64 el1 = 0, el2 = 0; int i = 0; do { @@ -511,7 +557,9 @@ static int feasible(const BMK_benchResult_t results, const constraint_t target) * bonus to exceeding the constraint value. We also give linear ratio for compression ratio. * The constant factors are experimental. */ -static double resultScore(const BMK_benchResult_t res, const size_t srcSize, const constraint_t target) { +static double +resultScore(const BMK_benchResult_t res, const size_t srcSize, const constraint_t target) +{ double cs = 0., ds = 0., rt, cm = 0.; const double r1 = 1, r2 = 0.1, rtr = 0.5; double ret; @@ -527,7 +575,9 @@ static double resultScore(const BMK_benchResult_t res, const size_t srcSize, con } /* calculates normalized squared euclidean distance of result1 if it is in the first quadrant relative to lvlRes */ -static double resultDistLvl(const BMK_benchResult_t result1, const BMK_benchResult_t lvlRes) { +static double +resultDistLvl(const BMK_benchResult_t result1, const BMK_benchResult_t lvlRes) +{ double normalizedCSpeedGain1 = (result1.cSpeed / lvlRes.cSpeed) - 1; double normalizedRatioGain1 = ((double)lvlRes.cSize / result1.cSize) - 1; if(normalizedRatioGain1 < 0 || normalizedCSpeedGain1 < 0) { @@ -537,16 +587,21 @@ static double resultDistLvl(const BMK_benchResult_t result1, const BMK_benchResu } /* return true if r2 strictly better than r1 */ -static int compareResultLT(const BMK_benchResult_t result1, const BMK_benchResult_t result2, const constraint_t target, size_t srcSize) { +static int +compareResultLT(const BMK_benchResult_t result1, const BMK_benchResult_t result2, const constraint_t target, size_t srcSize) +{ if(feasible(result1, target) && feasible(result2, target)) { if(g_optmode) { return resultDistLvl(result1, g_lvltarget) < resultDistLvl(result2, g_lvltarget); } else { - return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed) - || (result1.cSize == result2.cSize && result2.cSpeed == result1.cSpeed && result2.dSpeed > result1.dSpeed); + return (result1.cSize > result2.cSize) + || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed) + || (result1.cSize == result2.cSize && result2.cSpeed == result1.cSpeed && result2.dSpeed > result1.dSpeed); } } - return feasible(result2, target) || (!feasible(result1, target) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target))); + return feasible(result2, target) + || (!feasible(result1, target) + && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target))); } static constraint_t relaxTarget(constraint_t target) { @@ -556,14 +611,17 @@ static constraint_t relaxTarget(constraint_t target) { return target; } -static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) { +static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) +{ varInds_t v; for(v = 0; v < NUM_PARAMS; v++) { if(pc->vals[v] != PARAM_UNSET) { U32 newval = MIN(MAX(pc->vals[v], mintable[v]), maxtable[v]); if(newval != pc->vals[v]) { pc->vals[v] = newval; - DISPLAY("Warning: parameter %s not in valid range, adjusting to ", g_paramNames[v]); displayParamVal(stderr, v, newval, 0); DISPLAY("\n"); + DISPLAY("Warning: parameter %s not in valid range, adjusting to ", + g_paramNames[v]); + displayParamVal(stderr, v, newval, 0); DISPLAY("\n"); } } } @@ -577,7 +635,8 @@ static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) { U32 adjust = MAX(mintable[wlog_ind], sshb); if(adjust != pc->vals[wlog_ind]) { pc->vals[wlog_ind] = adjust; - DISPLAY("Warning: windowLog larger than src/block size, adjusted to %u\n", pc->vals[wlog_ind]); + DISPLAY("Warning: windowLog larger than src/block size, adjusted to %u\n", + (unsigned)pc->vals[wlog_ind]); } } } @@ -592,40 +651,52 @@ static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) { if(pc->vals[clog_ind] > maxclog) { pc->vals[clog_ind] = maxclog; - DISPLAY("Warning: chainlog too much larger than windowLog size, adjusted to %u\n", pc->vals[clog_ind]); + DISPLAY("Warning: chainlog too much larger than windowLog size, adjusted to %u\n", + (unsigned)pc->vals[clog_ind]); } } if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[hlog_ind] != PARAM_UNSET) { if(pc->vals[wlog_ind] + 1 < pc->vals[hlog_ind]) { pc->vals[hlog_ind] = pc->vals[wlog_ind] + 1; - DISPLAY("Warning: hashlog too much larger than windowLog size, adjusted to %u\n", pc->vals[hlog_ind]); + DISPLAY("Warning: hashlog too much larger than windowLog size, adjusted to %u\n", + (unsigned)pc->vals[hlog_ind]); } } if(pc->vals[slog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) { if(pc->vals[slog_ind] > pc->vals[clog_ind]) { pc->vals[clog_ind] = pc->vals[slog_ind]; - DISPLAY("Warning: searchLog larger than chainLog, adjusted to %u\n", pc->vals[slog_ind]); + DISPLAY("Warning: searchLog larger than chainLog, adjusted to %u\n", + (unsigned)pc->vals[slog_ind]); } } } -static int redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t maxBlockSize) { +static int +redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t maxBlockSize) +{ return (ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramValues)) > (size_t)target.cMem) /* Uses too much memory */ || ((1ULL << (paramValues.vals[wlog_ind] - 1)) >= maxBlockSize && paramValues.vals[wlog_ind] != mintable[wlog_ind]) /* wlog too much bigger than src size */ || (paramValues.vals[clog_ind] > (paramValues.vals[wlog_ind] + (paramValues.vals[strt_ind] > ZSTD_btlazy2))) /* chainLog larger than windowLog*/ || (paramValues.vals[slog_ind] > paramValues.vals[clog_ind]) /* searchLog larger than chainLog */ || (paramValues.vals[hlog_ind] > paramValues.vals[wlog_ind] + 1); /* hashLog larger than windowLog + 1 */ - } + /*-************************************ * Display Functions **************************************/ -static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) { +/* BMK_paramValues_into_commandLine() : + * transform a set of parameters paramValues_t + * into a command line compatible with `zstd` syntax + * and writes it into FILE* f. + * f must be already opened and writable */ +static void +BMK_paramValues_into_commandLine(FILE* f, const paramValues_t params) +{ varInds_t v; int first = 1; fprintf(f,"--zstd="); @@ -634,60 +705,13 @@ static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) { if (!first) { fprintf(f, ","); } fprintf(f,"%s=", g_paramNames[v]); - if (v == strt_ind) { fprintf(f,"%u", params.vals[v]); } + if (v == strt_ind) { fprintf(f,"%u", (unsigned)params.vals[v]); } else { displayParamVal(f, v, params.vals[v], 0); } first = 0; } fprintf(f, "\n"); } -static void BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize) -{ - varInds_t v; - int first = 1; - res.params = cParamUnsetMin(res.params); - fprintf(f, " {"); - for (v = 0; v < NUM_PARAMS; v++) { - if (g_silenceParams[v]) { continue; } - if (!first) { fprintf(f, ","); } - displayParamVal(f, v, res.params.vals[v], 3); - first = 0; - } - - { double const ratio = res.result.cSize ? - (double)srcSize / res.result.cSize : 0; - double const cSpeedMBps = (double)res.result.cSpeed / MB_UNIT; - double const dSpeedMBps = (double)res.result.dSpeed / MB_UNIT; - - fprintf(f, " }, /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", - ratio, cSpeedMBps, dSpeedMBps); - } -} - -/* Writes to f the results of a parameter benchmark */ -/* when used with --optimize, will only print results better than previously discovered */ -static void BMK_printWinner(FILE* f, const int cLevel, const BMK_benchResult_t result, const paramValues_t params, const size_t srcSize) -{ - char lvlstr[15] = "Custom Level"; - winnerInfo_t w; - w.params = params; - w.result = result; - - fprintf(f, "\r%79s\r", ""); - - if(cLevel != CUSTOM_LEVEL) { - snprintf(lvlstr, 15, " Level %2d ", cLevel); - } - - if(TIMED) { - const U64 time = UTIL_clockSpanNano(g_time); - const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); - fprintf(f, "%1lu:%2lu:%05.2f - ", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); - } - - fprintf(f, "/* %s */ ", lvlstr); - BMK_displayOneResult(f, w, srcSize); -} /* comparison function: */ /* strictly better, strictly worse, equal, speed-side adv, size-side adv */ @@ -698,7 +722,9 @@ static void BMK_printWinner(FILE* f, const int cLevel, const BMK_benchResult_t r #define SPEED_RESULT 4 #define SIZE_RESULT 5 /* maybe have epsilon-eq to limit table size? */ -static int speedSizeCompare(const BMK_benchResult_t r1, const BMK_benchResult_t r2) { +static int +speedSizeCompare(const BMK_benchResult_t r1, const BMK_benchResult_t r2) +{ if(r1.cSpeed < r2.cSpeed) { if(r1.cSize >= r2.cSize) { return BETTER_RESULT; @@ -714,7 +740,9 @@ static int speedSizeCompare(const BMK_benchResult_t r1, const BMK_benchResult_t /* 0 for insertion, 1 for no insert */ /* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */ -static int insertWinner(const winnerInfo_t w, const constraint_t targetConstraints) { +static int +insertWinner(const winnerInfo_t w, const constraint_t targetConstraints) +{ BMK_benchResult_t r = w.result; winner_ll_node* cur_node = g_winners; /* first node to insert */ @@ -808,20 +836,82 @@ static int insertWinner(const winnerInfo_t w, const constraint_t targetConstrain } } -static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_benchResult_t result, const paramValues_t params, const constraint_t targetConstraints, const size_t srcSize) +static void +BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize) +{ + varInds_t v; + int first = 1; + res.params = cParamUnsetMin(res.params); + fprintf(f, " {"); + for (v = 0; v < NUM_PARAMS; v++) { + if (g_silenceParams[v]) { continue; } + if (!first) { fprintf(f, ","); } + displayParamVal(f, v, res.params.vals[v], 3); + first = 0; + } + + { double const ratio = res.result.cSize ? + (double)srcSize / res.result.cSize : 0; + double const cSpeedMBps = (double)res.result.cSpeed / MB_UNIT; + double const dSpeedMBps = (double)res.result.dSpeed / MB_UNIT; + + fprintf(f, " }, /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", + ratio, cSpeedMBps, dSpeedMBps); + } +} + +/* Writes to f the results of a parameter benchmark */ +/* when used with --optimize, will only print results better than previously discovered */ +static void +BMK_printWinner(FILE* f, const int cLevel, const BMK_benchResult_t result, const paramValues_t params, const size_t srcSize) +{ + char lvlstr[15] = "Custom Level"; + winnerInfo_t w; + w.params = params; + w.result = result; + + fprintf(f, "\r%79s\r", ""); + + if(cLevel != CUSTOM_LEVEL) { + snprintf(lvlstr, 15, " Level %2d ", cLevel); + } + + if(TIMED) { + const U64 mn_in_ns = 60ULL * TIMELOOP_NANOSEC; + const U64 time_ns = UTIL_clockSpanNano(g_time); + const U64 minutes = time_ns / mn_in_ns; + fprintf(f, "%1lu:%2lu:%05.2f - ", + (unsigned long) minutes / 60, + (unsigned long) minutes % 60, + (double)(time_ns - (minutes * mn_in_ns)) / TIMELOOP_NANOSEC ); + } + + fprintf(f, "/* %s */ ", lvlstr); + BMK_displayOneResult(f, w, srcSize); +} + +static void +BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_benchResult_t result, const paramValues_t params, const constraint_t targetConstraints, const size_t srcSize) { /* global winner used for constraints */ /* cSize, cSpeed, dSpeed, cMem */ - static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL }, { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } }; - if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize) || g_displayLevel >= 4) { - if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { + static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL }, + { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } + }; + if ( DEBUG + || compareResultLT(g_winner.result, result, targetConstraints, srcSize) + || g_displayLevel >= 4) { + if ( DEBUG + && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { DISPLAY("New Winner: \n"); } - if(g_displayLevel >= 2) { BMK_printWinner(f, cLevel, result, params, srcSize); } + if(g_displayLevel >= 2) { + BMK_printWinner(f, cLevel, result, params, srcSize); + } if(compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { - if(g_displayLevel >= 1) { BMK_translateAdvancedParams(f, params); } + if(g_displayLevel >= 1) { BMK_paramValues_into_commandLine(f, params); } g_winner.result = result; g_winner.params = params; } @@ -849,31 +939,88 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_benchResult_ fprintf(f, "Overall Winner: \n"); BMK_displayOneResult(f, g_winner, srcSize); - BMK_translateAdvancedParams(f, g_winner.params); + BMK_paramValues_into_commandLine(f, g_winner.params); fprintf(f, "Latest BMK: \n");\ BMK_displayOneResult(f, w, srcSize); } } -static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, const size_t srcSize) + +/* BMK_print_cLevelEntry() : + * Writes one cLevelTable entry, for one level. + * f must exist, be already opened, and be seekable. + * this function cannot error. + */ +static void +BMK_print_cLevelEntry(FILE* f, const int cLevel, + paramValues_t params, + const BMK_benchResult_t result, const size_t srcSize) +{ + varInds_t v; + int first = 1; + + assert(cLevel >= 0); + assert(cLevel <= NB_LEVELS_TRACKED); + params = cParamUnsetMin(params); + + fprintf(f, " {"); + /* print cParams. + * assumption : all cParams are present and in order in the following range */ + for (v = 0; v <= strt_ind; v++) { + if (!first) { fprintf(f, ","); } + displayParamVal(f, v, params.vals[v], 3); + first = 0; + } + /* print comment */ + { double const ratio = result.cSize ? + (double)srcSize / result.cSize : 0; + double const cSpeedMBps = (double)result.cSpeed / MB_UNIT; + double const dSpeedMBps = (double)result.dSpeed / MB_UNIT; + + fprintf(f, " }, /* level %2i: R=%5.3f at %5.1f MB/s - %5.1f MB/s */\n", + cLevel, ratio, cSpeedMBps, dSpeedMBps); + } +} + + +/* BMK_print_cLevelTable() : + * print candidate compression table into proposed FILE* f. + * f must exist, be already opened, and be seekable. + * winners must be a table of NB_LEVELS_TRACKED+1 elements winnerInfo_t, all entries presumed initialized + * this function cannot error. + */ +static void +BMK_print_cLevelTable(FILE* f, const winnerInfo_t* winners, const size_t srcSize) { int cLevel; fprintf(f, "\n /* Proposed configurations : */ \n"); - fprintf(f, " /* W, C, H, S, L, T, strat */ \n"); + fprintf(f, " /* W, C, H, S, L, T, strat */ \n"); for (cLevel=0; cLevel <= NB_LEVELS_TRACKED; cLevel++) - BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize); + BMK_print_cLevelEntry(f, + cLevel, winners[cLevel].params, + winners[cLevel].result, srcSize); } -static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, const size_t srcSize) +/* BMK_saveAndPrint_cLevelTable() : + * save candidate compression table into FILE* f, + * and then to stdout. + * f must exist, be already opened, and be seekable. + * winners must be a table of NB_LEVELS_TRACKED+1 elements winnerInfo_t, all entries presumed initialized + * this function cannot error. + */ +static void +BMK_saveAndPrint_cLevelTable(FILE* const f, + const winnerInfo_t* winners, + const size_t srcSize) { fseek(f, 0, SEEK_SET); - BMK_printWinners2(f, winners, srcSize); + BMK_print_cLevelTable(f, winners, srcSize); fflush(f); - BMK_printWinners2(stdout, winners, srcSize); + BMK_print_cLevelTable(stdout, winners, srcSize); } @@ -892,9 +1039,8 @@ typedef struct { static size_t local_initCCtx(void* payload) { const BMK_initCCtxArgs* ag = (const BMK_initCCtxArgs*)payload; varInds_t i; - ZSTD_CCtx_reset(ag->cctx); - ZSTD_CCtx_resetParameters(ag->cctx); - ZSTD_CCtx_setParameter(ag->cctx, ZSTD_p_compressionLevel, ag->cLevel); + ZSTD_CCtx_reset(ag->cctx, ZSTD_reset_session_and_parameters); + ZSTD_CCtx_setParameter(ag->cctx, ZSTD_c_compressionLevel, ag->cLevel); for(i = 0; i < NUM_PARAMS; i++) { if(ag->comprParams->vals[i] != PARAM_UNSET) @@ -913,37 +1059,20 @@ typedef struct { static size_t local_initDCtx(void* payload) { const BMK_initDCtxArgs* ag = (const BMK_initDCtxArgs*)payload; - ZSTD_DCtx_reset(ag->dctx); + ZSTD_DCtx_reset(ag->dctx, ZSTD_reset_session_and_parameters); ZSTD_DCtx_loadDictionary(ag->dctx, ag->dictBuffer, ag->dictBufferSize); return 0; } /* additional argument is just the context */ static size_t local_defaultCompress( - const void* srcBuffer, size_t srcSize, - void* dstBuffer, size_t dstSize, - void* addArgs) { - size_t moreToFlush = 1; - ZSTD_CCtx* ctx = (ZSTD_CCtx*)addArgs; - ZSTD_inBuffer in; - ZSTD_outBuffer out; - in.src = srcBuffer; - in.size = srcSize; - in.pos = 0; - out.dst = dstBuffer; - out.size = dstSize; - out.pos = 0; + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + void* addArgs) +{ + ZSTD_CCtx* cctx = (ZSTD_CCtx*)addArgs; assert(dstSize == ZSTD_compressBound(srcSize)); /* specific to this version, which is only used in paramgrill */ - while (moreToFlush) { - if(out.pos == out.size) { - return (size_t)-ZSTD_error_dstSize_tooSmall; - } - moreToFlush = ZSTD_compress_generic(ctx, &out, &in, ZSTD_e_end); - if (ZSTD_isError(moreToFlush)) { - return moreToFlush; - } - } - return out.pos; + return ZSTD_compress2(cctx, dstBuffer, dstSize, srcBuffer, srcSize); } /* additional argument is just the context */ @@ -965,7 +1094,7 @@ static size_t local_defaultDecompress( if(out.pos == out.size) { return (size_t)-ZSTD_error_dstSize_tooSmall; } - moreToFlush = ZSTD_decompress_generic(dctx, + moreToFlush = ZSTD_decompressStream(dctx, &out, &in); if (ZSTD_isError(moreToFlush)) { return moreToFlush; @@ -1235,7 +1364,7 @@ static size_t sanitizeVarArray(varInds_t* varNew, const size_t varLength, const if( !((varArray[i] == clog_ind && strat == ZSTD_fast) || (varArray[i] == slog_ind && strat == ZSTD_fast) || (varArray[i] == slog_ind && strat == ZSTD_dfast) - || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) { + || (varArray[i] == tlen_ind && strat < ZSTD_btopt && strat != ZSTD_fast))) { varNew[j] = varArray[i]; j++; } @@ -1307,10 +1436,12 @@ static void memoTableSet(const memoTable_t* memoTableArray, const paramValues_t } /* frees all allocated memotables */ +/* secret contract : + * mtAll is a table of (ZSTD_STRATEGY_MAX+1) memoTable_t */ static void freeMemoTableArray(memoTable_t* const mtAll) { int i; if(mtAll == NULL) { return; } - for(i = 1; i <= (int)ZSTD_btultra; i++) { + for(i = 1; i <= (int)ZSTD_STRATEGY_MAX; i++) { free(mtAll[i].table); } free(mtAll); @@ -1318,21 +1449,26 @@ static void freeMemoTableArray(memoTable_t* const mtAll) { /* inits memotables for all (including mallocs), all strategies */ /* takes unsanitized varyParams */ -static memoTable_t* createMemoTableArray(const paramValues_t p, const varInds_t* const varyParams, const size_t varyLen, const U32 memoTableLog) { - memoTable_t* mtAll = (memoTable_t*)calloc(sizeof(memoTable_t),(ZSTD_btultra + 1)); - ZSTD_strategy i, stratMin = ZSTD_fast, stratMax = ZSTD_btultra; +static memoTable_t* +createMemoTableArray(const paramValues_t p, + const varInds_t* const varyParams, + const size_t varyLen, + const U32 memoTableLog) +{ + memoTable_t* const mtAll = (memoTable_t*)calloc(sizeof(memoTable_t),(ZSTD_STRATEGY_MAX + 1)); + ZSTD_strategy i, stratMin = ZSTD_STRATEGY_MIN, stratMax = ZSTD_STRATEGY_MAX; if(mtAll == NULL) { return NULL; } - for(i = 1; i <= (int)ZSTD_btultra; i++) { + for(i = 1; i <= (int)ZSTD_STRATEGY_MAX; i++) { mtAll[i].varLen = sanitizeVarArray(mtAll[i].varArray, varyLen, varyParams, i); } /* no memoization */ if(memoTableLog == 0) { - for(i = 1; i <= (int)ZSTD_btultra; i++) { + for(i = 1; i <= (int)ZSTD_STRATEGY_MAX; i++) { mtAll[i].tableType = noMemo; mtAll[i].table = NULL; mtAll[i].tableLen = 0; @@ -1391,13 +1527,24 @@ static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTa * Benchmarking Functions **************************************/ +static void display_params_tested(paramValues_t cParams) +{ + varInds_t vi; + DISPLAYLEVEL(3, "\r testing :"); + for (vi=0; vi < NUM_PARAMS; vi++) { + DISPLAYLEVEL(3, "%3u,", (unsigned)cParams.vals[vi]); + } + DISPLAYLEVEL(3, "\b \r"); +} + /* Replicate functionality of benchMemAdvanced, but with pre-split src / dst buffers */ /* The purpose is so that sufficient information is returned so that a decompression call to benchMemInvertible is possible */ /* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */ /* nbSeconds used in same way as in BMK_advancedParams_t */ /* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs */ /* dictionary nullable, nothing else though. */ -/* note : it would be better if this function was in bench.c, sharing code with benchMemAdvanced(), since it's technically a part of it */ +/* note : it would be a lot better if this function was present in benchzstd.c, + * sharing code with benchMemAdvanced(), since it's technically a part of it */ static BMK_benchOutcome_t BMK_benchMemInvertible( buffers_t buf, contexts_t ctx, int cLevel, const paramValues_t* comprParams, @@ -1420,6 +1567,7 @@ BMK_benchMemInvertible( buffers_t buf, contexts_t ctx, ZSTD_DCtx* dctx = ctx.dctx; /* init */ + display_params_tested(*comprParams); memset(&bResult, 0, sizeof(bResult)); /* warmimg up memory */ @@ -1438,13 +1586,40 @@ BMK_benchMemInvertible( buffers_t buf, contexts_t ctx, int decompressionCompleted = (mode == BMK_compressOnly); BMK_timedFnState_t* timeStateCompress = BMK_createTimedFnState(nbSeconds * 1000, 1000); BMK_timedFnState_t* timeStateDecompress = BMK_createTimedFnState(nbSeconds * 1000, 1000); + BMK_benchParams_t cbp, dbp; BMK_initCCtxArgs cctxprep; BMK_initDCtxArgs dctxprep; + + cbp.benchFn = local_defaultCompress; + cbp.benchPayload = cctx; + cbp.initFn = local_initCCtx; + cbp.initPayload = &cctxprep; + cbp.errorFn = ZSTD_isError; + cbp.blockCount = nbBlocks; + cbp.srcBuffers = srcPtrs; + cbp.srcSizes = srcSizes; + cbp.dstBuffers = dstPtrs; + cbp.dstCapacities = dstCapacities; + cbp.blockResults = dstSizes; + cctxprep.cctx = cctx; cctxprep.dictBuffer = dictBuffer; cctxprep.dictBufferSize = dictBufferSize; cctxprep.cLevel = cLevel; cctxprep.comprParams = comprParams; + + dbp.benchFn = local_defaultDecompress; + dbp.benchPayload = dctx; + dbp.initFn = local_initDCtx; + dbp.initPayload = &dctxprep; + dbp.errorFn = ZSTD_isError; + dbp.blockCount = nbBlocks; + dbp.srcBuffers = (const void* const *) dstPtrs; + dbp.srcSizes = dstCapacities; + dbp.dstBuffers = resPtrs; + dbp.dstCapacities = resSizes; + dbp.blockResults = NULL; + dctxprep.dctx = dctx; dctxprep.dictBuffer = dictBuffer; dctxprep.dictBufferSize = dictBufferSize; @@ -1452,13 +1627,7 @@ BMK_benchMemInvertible( buffers_t buf, contexts_t ctx, assert(timeStateCompress != NULL); assert(timeStateDecompress != NULL); while(!compressionCompleted) { - BMK_runOutcome_t const cOutcome = BMK_benchTimedFn(timeStateCompress, - &local_defaultCompress, cctx, - &local_initCCtx, &cctxprep, - nbBlocks, - srcPtrs, srcSizes, - dstPtrs, dstCapacities, - dstSizes); + BMK_runOutcome_t const cOutcome = BMK_benchTimedFn(timeStateCompress, cbp); if (!BMK_isSuccessful_runOutcome(cOutcome)) { BMK_benchOutcome_t bOut; @@ -1476,13 +1645,7 @@ BMK_benchMemInvertible( buffers_t buf, contexts_t ctx, } while (!decompressionCompleted) { - BMK_runOutcome_t const dOutcome = BMK_benchTimedFn(timeStateDecompress, - &local_defaultDecompress, dctx, - &local_initDCtx, &dctxprep, - nbBlocks, - (const void* const*)dstPtrs, dstSizes, - resPtrs, resSizes, - NULL); + BMK_runOutcome_t const dOutcome = BMK_benchTimedFn(timeStateDecompress, dbp); if (!BMK_isSuccessful_runOutcome(dOutcome)) { BMK_benchOutcome_t bOut; @@ -1512,6 +1675,10 @@ BMK_benchMemInvertible( buffers_t buf, contexts_t ctx, } } +/* BMK_benchParam() : + * benchmark a set of `cParams` over sample `buf`, + * store the result in `resultPtr`. + * @return : 0 if success, 1 if error */ static int BMK_benchParam ( BMK_benchResult_t* resultPtr, buffers_t buf, contexts_t ctx, paramValues_t cParams) @@ -1519,30 +1686,12 @@ static int BMK_benchParam ( BMK_benchResult_t* resultPtr, BMK_benchOutcome_t const outcome = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, 3); - int const success = BMK_isSuccessful_benchOutcome(outcome); - if (!success) return 1; + if (!BMK_isSuccessful_benchOutcome(outcome)) return 1; *resultPtr = BMK_extract_benchResult(outcome); return 0; } -#define CBENCHMARK(conditional, resultvar, tmpret, mode, sec) { \ - if(conditional) { \ - BMK_benchOutcome_t const outcome = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, mode, sec); \ - if (!BMK_isSuccessful_benchOutcome(outcome)) { \ - DEBUGOUTPUT("Benchmarking failed\n"); \ - return ERROR_RESULT; \ - } \ - { BMK_benchResult_t const tmpResult = BMK_extract_benchResult(outcome); \ - if (mode != BMK_decodeOnly) { \ - resultvar.cSpeed = tmpResult.cSpeed; \ - resultvar.cSize = tmpResult.cSize; \ - resultvar.cMem = tmpResult.cMem; \ - } \ - if (mode != BMK_compressOnly) { resultvar.dSpeed = tmpResult.dSpeed; } \ - } } \ -} - /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */ #define VARIANCE 1.2 static int allBench(BMK_benchResult_t* resultPtr, @@ -1662,12 +1811,14 @@ static void BMK_init_level_constraints(int bytePerSec_level1) g_level_constraint[l].cSpeed_min = (g_level_constraint[l-1].cSpeed_min * 49) / 64; g_level_constraint[l].dSpeed_min = 0.; g_level_constraint[l].windowLog_max = (l<20) ? 23 : l+5; /* only --ultra levels >= 20 can use windowlog > 23 */ - g_level_constraint[l].strategy_max = (l<19) ? ZSTD_btopt : ZSTD_btultra; /* level 19 is allowed to use btultra */ + g_level_constraint[l].strategy_max = ZSTD_STRATEGY_MAX; } } } -static int BMK_seed(winnerInfo_t* winners, const paramValues_t params, - const buffers_t buf, const contexts_t ctx) +static int BMK_seed(winnerInfo_t* winners, + const paramValues_t params, + const buffers_t buf, + const contexts_t ctx) { BMK_benchResult_t testResult; int better = 0; @@ -1675,8 +1826,8 @@ static int BMK_seed(winnerInfo_t* winners, const paramValues_t params, BMK_benchParam(&testResult, buf, ctx, params); - for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) { + if (testResult.cSpeed < g_level_constraint[cLevel].cSpeed_min) continue; /* not fast enough for this level */ if (testResult.dSpeed < g_level_constraint[cLevel].dSpeed_min) @@ -1689,7 +1840,7 @@ static int BMK_seed(winnerInfo_t* winners, const paramValues_t params, /* first solution for this cLevel */ winners[cLevel].result = testResult; winners[cLevel].params = params; - BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize); + BMK_print_cLevelEntry(stdout, cLevel, params, testResult, buf.srcSize); better = 1; continue; } @@ -1719,7 +1870,7 @@ static int BMK_seed(winnerInfo_t* winners, const paramValues_t params, if (W_DMemUsed_note < O_DMemUsed_note) { /* uses too much Decompression memory for too little benefit */ if (W_ratio > O_ratio) - DISPLAY ("Decompression Memory : %5.3f @ %4.1f MB vs %5.3f @ %4.1f MB : not enough for level %i\n", + DISPLAYLEVEL(3, "Decompression Memory : %5.3f @ %4.1f MB vs %5.3f @ %4.1f MB : not enough for level %i\n", W_ratio, (double)(W_DMemUsed) / 1024 / 1024, O_ratio, (double)(O_DMemUsed) / 1024 / 1024, cLevel); continue; @@ -1727,34 +1878,38 @@ static int BMK_seed(winnerInfo_t* winners, const paramValues_t params, if (W_CMemUsed_note < O_CMemUsed_note) { /* uses too much memory for compression for too little benefit */ if (W_ratio > O_ratio) - DISPLAY ("Compression Memory : %5.3f @ %4.1f MB vs %5.3f @ %4.1f MB : not enough for level %i\n", + DISPLAYLEVEL(3, "Compression Memory : %5.3f @ %4.1f MB vs %5.3f @ %4.1f MB : not enough for level %i\n", W_ratio, (double)(W_CMemUsed) / 1024 / 1024, - O_ratio, (double)(O_CMemUsed) / 1024 / 1024, cLevel); + O_ratio, (double)(O_CMemUsed) / 1024 / 1024, + cLevel); continue; } if (W_CSpeed_note < O_CSpeed_note ) { /* too large compression speed difference for the compression benefit */ if (W_ratio > O_ratio) - DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", + DISPLAYLEVEL(3, "Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", W_ratio, (double)testResult.cSpeed / MB_UNIT, - O_ratio, (double)winners[cLevel].result.cSpeed / MB_UNIT, cLevel); + O_ratio, (double)winners[cLevel].result.cSpeed / MB_UNIT, + cLevel); continue; } if (W_DSpeed_note < O_DSpeed_note ) { /* too large decompression speed difference for the compression benefit */ if (W_ratio > O_ratio) - DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", + DISPLAYLEVEL(3, "Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", W_ratio, (double)testResult.dSpeed / MB_UNIT, - O_ratio, (double)winners[cLevel].result.dSpeed / MB_UNIT, cLevel); + O_ratio, (double)winners[cLevel].result.dSpeed / MB_UNIT, + cLevel); continue; } if (W_ratio < O_ratio) - DISPLAY("Solution %4.3f selected over %4.3f at level %i, due to better secondary statistics \n", W_ratio, O_ratio, cLevel); + DISPLAYLEVEL(3, "Solution %4.3f selected over %4.3f at level %i, due to better secondary statistics \n", + W_ratio, O_ratio, cLevel); winners[cLevel].result = testResult; winners[cLevel].params = params; - BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize); + BMK_print_cLevelEntry(stdout, cLevel, params, testResult, buf.srcSize); better = 1; } } @@ -1771,58 +1926,75 @@ static int BMK_seed(winnerInfo_t* winners, const paramValues_t params, #define PARAMTABLEMASK (PARAMTABLESIZE-1) static BYTE g_alreadyTested[PARAMTABLESIZE] = {0}; /* init to zero */ -static BYTE* NB_TESTS_PLAYED(paramValues_t p) { - ZSTD_compressionParameters p2 = pvalsToCParams(sanitizeParams(p)); - return &g_alreadyTested[(XXH64((void*)&p2, sizeof(p2), 0) >> 3) & PARAMTABLEMASK]; +static BYTE* NB_TESTS_PLAYED(paramValues_t p) +{ + ZSTD_compressionParameters const cParams = pvalsToCParams(sanitizeParams(p)); + unsigned long long const h64 = XXH64(&cParams, sizeof(cParams), 0); + return &g_alreadyTested[(h64 >> 3) & PARAMTABLEMASK]; } -static void playAround(FILE* f, winnerInfo_t* winners, +static void playAround(FILE* f, + winnerInfo_t* winners, paramValues_t p, const buffers_t buf, const contexts_t ctx) { - int nbVariations = 0, i; + int nbVariations = 0; UTIL_time_t const clockStart = UTIL_getTime(); while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) { - BYTE* b; - if (nbVariations++ > g_maxNbVariations) break; - do { for(i = 0; i < 4; i++) { paramVaryOnce(FUZ_rand(&g_rand) % (strt_ind + 1), ((FUZ_rand(&g_rand) & 1) << 1) - 1, &p); } } - while(!paramValid(p)); + do { + int i; + for(i = 0; i < 4; i++) { + paramVaryOnce(FUZ_rand(&g_rand) % (strt_ind + 1), + ((FUZ_rand(&g_rand) & 1) << 1) - 1, + &p); + } + } while (!paramValid(p)); /* exclude faster if already played params */ if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1)) continue; /* test */ - b = NB_TESTS_PLAYED(p); - (*b)++; + { BYTE* const b = NB_TESTS_PLAYED(p); + (*b)++; + } if (!BMK_seed(winners, p, buf, ctx)) continue; /* improvement found => search more */ - BMK_printWinners(f, winners, buf.srcSize); + BMK_saveAndPrint_cLevelTable(f, winners, buf.srcSize); playAround(f, winners, p, buf, ctx); } } -static void BMK_selectRandomStart( - FILE* f, winnerInfo_t* winners, +static void +BMK_selectRandomStart( FILE* f, + winnerInfo_t* winners, const buffers_t buf, const contexts_t ctx) { U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1); if ((id==0) || (winners[id].params.vals[wlog_ind]==0)) { /* use some random entry */ paramValues_t const p = adjustParams(cParamsToPVals(pvalsToCParams(randomParams())), /* defaults nonCompression parameters */ - buf.srcSize, 0); + buf.srcSize, 0); playAround(f, winners, p, buf, ctx); } else { playAround(f, winners, winners[id].params, buf, ctx); } } -static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx) + +/* BMK_generate_cLevelTable() : + * test a large number of configurations + * and distribute them accross compression levels according to speed conditions. + * display and save all intermediate results into rfName = "grillResults.txt". + * the function automatically stops after g_timeLimit_s. + * this function cannot error, it directly exit() in case of problem. + */ +static void BMK_generate_cLevelTable(const buffers_t buf, const contexts_t ctx) { paramValues_t params; winnerInfo_t winners[NB_LEVELS_TRACKED+1]; @@ -1851,17 +2023,17 @@ static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx) params = cParamsToPVals(ZSTD_getCParams(i, buf.maxBlockSize, 0)); BMK_seed(winners, params, buf, ctx); } } - BMK_printWinners(f, winners, buf.srcSize); + BMK_saveAndPrint_cLevelTable(f, winners, buf.srcSize); /* start tests */ { const UTIL_time_t grillStart = UTIL_getTime(); do { BMK_selectRandomStart(f, winners, buf, ctx); - } while (BMK_timeSpan(grillStart) < g_timeLimit_s); + } while (BMK_timeSpan_s(grillStart) < g_timeLimit_s); } /* end summary */ - BMK_printWinners(f, winners, buf.srcSize); + BMK_saveAndPrint_cLevelTable(f, winners, buf.srcSize); DISPLAY("grillParams operations completed \n"); /* clean up*/ @@ -1873,7 +2045,9 @@ static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx) * Single Benchmark Functions **************************************/ -static int benchOnce(const buffers_t buf, const contexts_t ctx, const int cLevel) { +static int +benchOnce(const buffers_t buf, const contexts_t ctx, const int cLevel) +{ BMK_benchResult_t testResult; g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize); @@ -1923,7 +2097,7 @@ static int benchSample(double compressibility, int cLevel) if(g_singleRun) { ret = benchOnce(buf, ctx, cLevel); } else { - BMK_benchFullTable(buf, ctx); + BMK_generate_cLevelTable(buf, ctx); } freeBuffers(buf); @@ -1963,7 +2137,7 @@ static int benchFiles(const char** fileNamesTable, int nbFiles, if (g_singleRun) { ret = benchOnce(buf, ctx, cLevel); } else { - BMK_benchFullTable(buf, ctx); + BMK_generate_cLevelTable(buf, ctx); } freeBuffers(buf); @@ -2040,15 +2214,15 @@ static winnerInfo_t climbOnce(const constraint_t target, } } } - } + } /* for (offset = -1; offset <= 1; offset += 2) */ } /* for (i = 0; i < varLen; i++) */ if(better) { continue; } - for(dist = 2; dist < varLen + 2; dist++) { /* varLen is # dimensions */ - for(i = 0; i < (1 << varLen) / varLen + 2; i++) { + for (dist = 2; dist < varLen + 2; dist++) { /* varLen is # dimensions */ + for (i = 0; i < (1 << varLen) / varLen + 2; i++) { int res; CHECKTIME(winnerInfo); candidateInfo.params = cparam; @@ -2095,11 +2269,12 @@ static winnerInfo_t climbOnce(const constraint_t target, weight more on visit for bad results, less on good results/more on later results / ones with more failures. allocate memoTable here. */ -static winnerInfo_t optimizeFixedStrategy( - const buffers_t buf, const contexts_t ctx, - const constraint_t target, paramValues_t paramTarget, - const ZSTD_strategy strat, - memoTable_t* memoTableArray, const int tries) { +static winnerInfo_t +optimizeFixedStrategy(const buffers_t buf, const contexts_t ctx, + const constraint_t target, paramValues_t paramTarget, + const ZSTD_strategy strat, + memoTable_t* memoTableArray, const int tries) +{ int i = 0; paramValues_t init; @@ -2114,9 +2289,11 @@ static winnerInfo_t optimizeFixedStrategy( for(i = 0; i < tries; i++) { DEBUGOUTPUT("Restart\n"); - do { randomConstrainedParams(&init, memoTableArray, strat); } while(redundantParams(init, target, buf.maxBlockSize)); + do { + randomConstrainedParams(&init, memoTableArray, strat); + } while(redundantParams(init, target, buf.maxBlockSize)); candidateInfo = climbOnce(target, memoTableArray, buf, ctx, init); - if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) { + if (compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) { winnerInfo = candidateInfo; BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); i = 0; @@ -2130,12 +2307,13 @@ static winnerInfo_t optimizeFixedStrategy( /* goes best, best-1, best+1, best-2, ... */ /* return 0 if nothing remaining */ -static int nextStrategy(const int currentStrategy, const int bestStrategy) { +static int nextStrategy(const int currentStrategy, const int bestStrategy) +{ if(bestStrategy <= currentStrategy) { int candidate = 2 * bestStrategy - currentStrategy - 1; if(candidate < 1) { candidate = currentStrategy + 1; - if(candidate > (int)ZSTD_btultra) { + if(candidate > (int)ZSTD_STRATEGY_MAX) { return 0; } else { return candidate; @@ -2145,7 +2323,7 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) { } } else { /* bestStrategy >= currentStrategy */ int candidate = 2 * bestStrategy - currentStrategy; - if(candidate > (int)ZSTD_btultra) { + if(candidate > (int)ZSTD_STRATEGY_MAX) { candidate = currentStrategy - 1; if(candidate < 1) { return 0; @@ -2176,8 +2354,12 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) { static int g_maxTries = 5; #define TRY_DECAY 1 -static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, paramValues_t paramTarget, - const int cLevelOpt, const int cLevelRun, const U32 memoTableLog) +static int +optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, + const char* dictFileName, + constraint_t target, paramValues_t paramTarget, + const int cLevelOpt, const int cLevelRun, + const U32 memoTableLog) { varInds_t varArray [NUM_PARAMS]; int ret = 0; @@ -2189,18 +2371,18 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ buffers_t buf; g_time = UTIL_getTime(); - if(createBuffers(&buf, fileNamesTable, nbFiles)) { + if (createBuffers(&buf, fileNamesTable, nbFiles)) { DISPLAY("unable to load files\n"); return 1; } - if(createContexts(&ctx, dictFileName)) { + if (createContexts(&ctx, dictFileName)) { DISPLAY("unable to load dictionary\n"); freeBuffers(buf); return 2; } - if(nbFiles == 1) { + if (nbFiles == 1) { DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[0]); } else { DISPLAYLEVEL(2, "Loading %lu Files... \r", (unsigned long)nbFiles); @@ -2274,21 +2456,19 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ DISPLAYLEVEL(2, "optimizing for %lu Files", (unsigned long)nbFiles); } - if(target.cSpeed != 0) { DISPLAYLEVEL(2," - limit compression speed %u MB/s", target.cSpeed >> 20); } - if(target.dSpeed != 0) { DISPLAYLEVEL(2, " - limit decompression speed %u MB/s", target.dSpeed >> 20); } - if(target.cMem != (U32)-1) { DISPLAYLEVEL(2, " - limit memory %u MB", target.cMem >> 20); } + if(target.cSpeed != 0) { DISPLAYLEVEL(2," - limit compression speed %u MB/s", (unsigned)(target.cSpeed >> 20)); } + if(target.dSpeed != 0) { DISPLAYLEVEL(2, " - limit decompression speed %u MB/s", (unsigned)(target.dSpeed >> 20)); } + if(target.cMem != (U32)-1) { DISPLAYLEVEL(2, " - limit memory %u MB", (unsigned)(target.cMem >> 20)); } DISPLAYLEVEL(2, "\n"); - findClockGranularity(); + init_clockGranularity(); { paramValues_t CParams; /* find best solution from default params */ - { - /* strategy selection */ - const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); + { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); DEBUGOUTPUT("Strategy Selection\n"); - if(paramTarget.vals[strt_ind] == PARAM_UNSET) { + if (paramTarget.vals[strt_ind] == PARAM_UNSET) { BMK_benchResult_t candidate; int i; for (i=1; i<=maxSeeds; i++) { @@ -2313,16 +2493,14 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ DEBUGOUTPUT("Real Opt\n"); /* start 'real' optimization */ - { - int bestStrategy = (int)winner.params.vals[strt_ind]; - if(paramTarget.vals[strt_ind] == PARAM_UNSET) { + { int bestStrategy = (int)winner.params.vals[strt_ind]; + if (paramTarget.vals[strt_ind] == PARAM_UNSET) { int st = bestStrategy; int tries = g_maxTries; - { - /* one iterations of hill climbing with the level-defined parameters. */ - winnerInfo_t w1 = climbOnce(target, allMT, buf, ctx, winner.params); - if(compareResultLT(winner.result, w1.result, target, buf.srcSize)) { + /* one iterations of hill climbing with the level-defined parameters. */ + { winnerInfo_t const w1 = climbOnce(target, allMT, buf, ctx, winner.params); + if (compareResultLT(winner.result, w1.result, target, buf.srcSize)) { winner = w1; } CHECKTIMEGT(ret, 0, _displayCleanUp); @@ -2356,13 +2534,16 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ DISPLAY("No feasible solution found\n"); goto _cleanUp; } + /* end summary */ _displayCleanUp: - if(g_displayLevel >= 0) { BMK_displayOneResult(stdout, winner, buf.srcSize); } - BMK_translateAdvancedParams(stdout, winner.params); + if (g_displayLevel >= 0) { + BMK_displayOneResult(stdout, winner, buf.srcSize); + } + BMK_paramValues_into_commandLine(stdout, winner.params); DISPLAYLEVEL(1, "grillParams size - optimizer completed \n"); - } + _cleanUp: freeContexts(ctx); freeBuffers(buf); @@ -2459,7 +2640,8 @@ static int usage_advanced(void) DISPLAY( " -S : Single run \n"); DISPLAY( " --zstd : Single run, parameter selection same as zstdcli \n"); DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100); - DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", g_timeLimit_s, (double)g_timeLimit_s / 3600); + DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", + (unsigned)g_timeLimit_s, (double)g_timeLimit_s / 3600); DISPLAY( " -v : Prints Benchmarking output\n"); DISPLAY( " -D : Next argument dictionary file\n"); DISPLAY( " -s : Seperate Files\n"); @@ -2473,14 +2655,23 @@ static int badusage(const char* exename) return 1; } -#define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } } +#define PARSE_SUB_ARGS(stringLong, stringShort, variable) { \ + if ( longCommandWArg(&argument, stringLong) \ + || longCommandWArg(&argument, stringShort) ) { \ + variable = readU32FromChar(&argument); \ + if (argument[0]==',') { \ + argument++; continue; \ + } else break; \ +} } + /* 1 if successful parse, 0 otherwise */ static int parse_params(const char** argptr, paramValues_t* pv) { int matched = 0; const char* argOrig = *argptr; varInds_t v; for(v = 0; v < NUM_PARAMS; v++) { - if(longCommandWArg(argptr,g_shortParamNames[v]) || longCommandWArg(argptr, g_paramNames[v])) { + if ( longCommandWArg(argptr,g_shortParamNames[v]) + || longCommandWArg(argptr, g_paramNames[v]) ) { if(**argptr == '=') { (*argptr)++; pv->vals[v] = readU32FromChar(argptr); @@ -2648,7 +2839,7 @@ int main(int argc, const char** argv) continue; case 'l': /* search length */ argument++; - g_params.vals[slen_ind] = readU32FromChar(&argument); + g_params.vals[mml_ind] = readU32FromChar(&argument); continue; case 't': /* target length */ argument++; @@ -2685,7 +2876,7 @@ int main(int argc, const char** argv) case 'B': argument++; g_blockSize = readU32FromChar(&argument); - DISPLAY("using %u KB block size \n", g_blockSize>>10); + DISPLAY("using %u KB block size \n", (unsigned)(g_blockSize>>10)); break; /* caps runtime (in seconds) */ diff --git a/tests/playTests.sh b/tests/playTests.sh index b86a0dc40cb2..ef4861b52917 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -122,6 +122,18 @@ $ZSTD --fast=5000000000 -f tmp && die "too large numeric value : must fail" $ZSTD -c --fast=0 tmp > $INTOVOID && die "--fast must not accept value 0" $ECHO "test : too large numeric argument" $ZSTD --fast=9999999999 -f tmp && die "should have refused numeric value" +$ECHO "test : set compression level with environment variable ZSTD_CLEVEL" +ZSTD_CLEVEL=12 $ZSTD -f tmp # positive compression level +ZSTD_CLEVEL=-12 $ZSTD -f tmp # negative compression level +ZSTD_CLEVEL=+12 $ZSTD -f tmp # valid: verbose '+' sign +ZSTD_CLEVEL= $ZSTD -f tmp # empty env var, warn and revert to default setting +ZSTD_CLEVEL=- $ZSTD -f tmp # malformed env var, warn and revert to default setting +ZSTD_CLEVEL=a $ZSTD -f tmp # malformed env var, warn and revert to default setting +ZSTD_CLEVEL=+a $ZSTD -f tmp # malformed env var, warn and revert to default setting +ZSTD_CLEVEL=3a7 $ZSTD -f tmp # malformed env var, warn and revert to default setting +ZSTD_CLEVEL=50000000000 $ZSTD -f tmp # numeric value too large, warn and revert to default setting +$ECHO "test : override ZSTD_CLEVEL with command line option" +ZSTD_CLEVEL=12 $ZSTD --fast=3 -f tmp # overridden by command line option $ECHO "test : compress to stdout" $ZSTD tmp -c > tmpCompressed $ZSTD tmp --stdout > tmpCompressed # long command format @@ -179,8 +191,15 @@ $ECHO foo > tmpro chmod 400 tmpro.zst $ZSTD -q tmpro && die "should have refused to overwrite read-only file" $ZSTD -q -f tmpro +$ECHO "test: --no-progress flag" +$ZSTD tmpro -c --no-progress | $ZSTD -d -f -o "$INTOVOID" --no-progress +$ZSTD tmpro -cv --no-progress | $ZSTD -dv -f -o "$INTOVOID" --no-progress rm -f tmpro tmpro.zst - +$ECHO "test: overwrite input file (must fail)" +$ZSTD tmp -fo tmp && die "zstd compression overwrote the input file" +$ZSTD tmp.zst -dfo tmp.zst && die "zstd decompression overwrote the input file" +$ECHO "test: detect that input file does not exist" +$ZSTD nothere && die "zstd hasn't detected that input file does not exist" $ECHO "test : file removal" $ZSTD -f --rm tmp @@ -213,7 +232,7 @@ rm tmp* $ECHO "test : compress multiple files" $ECHO hello > tmp1 $ECHO world > tmp2 -$ZSTD tmp1 tmp2 -o "$INTOVOID" +$ZSTD tmp1 tmp2 -o "$INTOVOID" -f $ZSTD tmp1 tmp2 -c | $ZSTD -t $ZSTD tmp1 tmp2 -o tmp.zst test ! -f tmp1.zst @@ -221,7 +240,7 @@ test ! -f tmp2.zst $ZSTD tmp1 tmp2 $ZSTD -t tmp1.zst tmp2.zst $ZSTD -dc tmp1.zst tmp2.zst -$ZSTD tmp1.zst tmp2.zst -o "$INTOVOID" +$ZSTD tmp1.zst tmp2.zst -o "$INTOVOID" -f $ZSTD -d tmp1.zst tmp2.zst -o tmp touch tmpexists $ZSTD tmp1 tmp2 -f -o tmpexists @@ -237,14 +256,15 @@ $ECHO "\n===> Advanced compression parameters " $ECHO "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!" $ECHO "Hello world!" | $ZSTD --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!" $ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!" +$ECHO "Hello world!" | $ZSTD --zstd=strategy=10 - -o tmp.zst && die "parameter out of bound not detected!" # > btultra2 : does not exist test ! -f tmp.zst # tmp.zst should not be created roundTripTest -g512K -roundTripTest -g512K " --zstd=slen=3,tlen=48,strat=6" +roundTripTest -g512K " --zstd=mml=3,tlen=48,strat=6" roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6" -roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6" -roundTripTest -g512K " --single-thread --long --zstd=ldmHashLog=20,ldmSearchLength=64,ldmBucketSizeLog=1,ldmHashEveryLog=7" -roundTripTest -g512K " --single-thread --long --zstd=ldmhlog=20,ldmslen=64,ldmblog=1,ldmhevery=7" -roundTripTest -g512K 19 +roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,minMatch=3,targetLength=48,strategy=6" +roundTripTest -g512K " --single-thread --long --zstd=ldmHashLog=20,ldmMinMatch=64,ldmBucketSizeLog=1,ldmHashRateLog=7" +roundTripTest -g512K " --single-thread --long --zstd=lhlog=20,lmml=64,lblog=1,lhrlog=7" +roundTripTest -g64K "19 --zstd=strat=9" # btultra2 $ECHO "\n===> Pass-Through mode " @@ -541,6 +561,9 @@ $ECHO "bench negative level" $ZSTD -bi0 --fast tmp1 $ECHO "with recursive and quiet modes" $ZSTD -rqi1b1e2 tmp1 +$ECHO "benchmark decompression only" +$ZSTD -f tmp1 +$ZSTD -b -d -i1 tmp1.zst $ECHO "\n===> zstd compatibility tests " @@ -744,17 +767,17 @@ then ./datagen -g2MB > tmp refSize=$($ZSTD tmp -6 -c --zstd=wlog=18 | wc -c) ov9Size=$($ZSTD tmp -6 -c --zstd=wlog=18,ovlog=9 | wc -c) - ov0Size=$($ZSTD tmp -6 -c --zstd=wlog=18,ovlog=0 | wc -c) + ov1Size=$($ZSTD tmp -6 -c --zstd=wlog=18,ovlog=1 | wc -c) if [ $refSize -eq $ov9Size ]; then echo ov9Size should be different from refSize exit 1 fi - if [ $refSize -eq $ov0Size ]; then - echo ov0Size should be different from refSize + if [ $refSize -eq $ov1Size ]; then + echo ov1Size should be different from refSize exit 1 fi - if [ $ov9Size -ge $ov0Size ]; then - echo ov9Size=$ov9Size should be smaller than ov0Size=$ov0Size + if [ $ov9Size -ge $ov1Size ]; then + echo ov9Size=$ov9Size should be smaller than ov1Size=$ov1Size exit 1 fi @@ -833,6 +856,12 @@ $ECHO "===> test: --adapt must fail on incoherent bounds " ./datagen > tmp $ZSTD -f -vv --adapt=min=10,max=9 tmp && die "--adapt must fail on incoherent bounds" +$ECHO "\n===> rsyncable mode " +roundTripTest -g10M " --rsyncable" +roundTripTest -g10M " --rsyncable -B100K" +$ECHO "===> test: --rsyncable must fail with --single-thread" +$ZSTD -f -vv --rsyncable --single-thread tmp && die "--rsyncable must fail with --single-thread" + if [ "$1" != "--test-large-data" ]; then $ECHO "Skipping large data tests" diff --git a/tests/poolTests.c b/tests/poolTests.c index 9661b5299e52..8b9a4700229d 100644 --- a/tests/poolTests.c +++ b/tests/poolTests.c @@ -30,7 +30,7 @@ struct data { size_t i; }; -void fn(void *opaque) { +static void fn(void *opaque) { struct data *data = (struct data *)opaque; ZSTD_pthread_mutex_lock(&data->mutex); data->data[data->i] = data->i; @@ -38,7 +38,7 @@ void fn(void *opaque) { ZSTD_pthread_mutex_unlock(&data->mutex); } -int testOrder(size_t numThreads, size_t queueSize) { +static int testOrder(size_t numThreads, size_t queueSize) { struct data data; POOL_ctx *ctx = POOL_create(numThreads, queueSize); ASSERT_TRUE(ctx); @@ -63,13 +63,13 @@ int testOrder(size_t numThreads, size_t queueSize) { /* --- test deadlocks --- */ -void waitFn(void *opaque) { +static void waitFn(void *opaque) { (void)opaque; UTIL_sleepMilli(1); } /* Tests for deadlock */ -int testWait(size_t numThreads, size_t queueSize) { +static int testWait(size_t numThreads, size_t queueSize) { struct data data; POOL_ctx *ctx = POOL_create(numThreads, queueSize); ASSERT_TRUE(ctx); @@ -92,7 +92,7 @@ typedef struct { ZSTD_pthread_cond_t cond; } poolTest_t; -void waitLongFn(void *opaque) { +static void waitLongFn(void *opaque) { poolTest_t* test = (poolTest_t*) opaque; UTIL_sleepMilli(10); ZSTD_pthread_mutex_lock(&test->mut); @@ -167,7 +167,7 @@ typedef struct { int val; } abruptEndCanary_t; -void waitIncFn(void *opaque) { +static void waitIncFn(void *opaque) { abruptEndCanary_t* test = (abruptEndCanary_t*) opaque; UTIL_sleepMilli(10); ZSTD_pthread_mutex_lock(&test->mut); diff --git a/tests/regression/Makefile b/tests/regression/Makefile new file mode 100644 index 000000000000..03e5b0af1d70 --- /dev/null +++ b/tests/regression/Makefile @@ -0,0 +1,58 @@ +# ################################################################ +# Copyright (c) 2015-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ################################################################ + +CFLAGS ?= -O3 + +CURL_CFLAGS := $(shell curl-config --cflags) +CURL_LDFLAGS := $(shell curl-config --libs) -pthread + +PROGDIR := ../../programs +LIBDIR := ../../lib +ZSTD_CPPFLAGS := -I$(PROGDIR) -I$(LIBDIR) -I$(LIBDIR)/common + +REGRESSION_CFLAGS = $(CFLAGS) $(CURL_CFLAGS) +REGRESSION_CPPFLAGS = $(CPPFLAGS) $(ZSTD_CPPFLAGS) +REGRESSION_LDFLAGS = $(LDFLAGS) $(CURL_LDFLAGS) + +all: test + +xxhash.o: $(LIBDIR)/common/xxhash.c $(LIBDIR)/common/xxhash.h + $(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@ + +util.o: $(PROGDIR)/util.c $(PROGDIR)/util.h + $(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@ + +data.o: data.c data.h $(PROGDIR)/util.h $(LIBDIR)/common/xxhash.h + $(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@ + +config.o: config.c config.h levels.h + $(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@ + +method.h: data.h config.h result.h + +method.o: method.c method.h + $(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@ + +result.o: result.c result.h + $(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@ + +test.o: test.c data.h config.h method.h + $(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@ + +libzstd.a: + $(MAKE) -C $(LIBDIR) libzstd.a-mt + cp $(LIBDIR)/libzstd.a . + +test: test.o data.o config.o util.o method.o result.o xxhash.o libzstd.a + $(CC) $^ $(REGRESSION_LDFLAGS) -o $@ + +.PHONY: clean +clean: + $(MAKE) -C $(LIBDIR) clean + $(RM) *.o *.a test diff --git a/tests/regression/config.c b/tests/regression/config.c new file mode 100644 index 000000000000..8a6103b2929e --- /dev/null +++ b/tests/regression/config.c @@ -0,0 +1,230 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "config.h" + +/* Define a config for each fast level we want to test with. */ +#define FAST_LEVEL(x) \ + param_value_t const level_fast##x##_param_values[] = { \ + {.param = ZSTD_c_compressionLevel, .value = -x}, \ + }; \ + config_t const level_fast##x = { \ + .name = "level -" #x, \ + .cli_args = "--fast=" #x, \ + .param_values = PARAM_VALUES(level_fast##x##_param_values), \ + }; \ + config_t const level_fast##x##_dict = { \ + .name = "level -" #x " with dict", \ + .cli_args = "--fast=" #x, \ + .param_values = PARAM_VALUES(level_fast##x##_param_values), \ + .use_dictionary = 1, \ + }; + +/* Define a config for each level we want to test with. */ +#define LEVEL(x) \ + param_value_t const level_##x##_param_values[] = { \ + {.param = ZSTD_c_compressionLevel, .value = x}, \ + }; \ + config_t const level_##x = { \ + .name = "level " #x, \ + .cli_args = "-" #x, \ + .param_values = PARAM_VALUES(level_##x##_param_values), \ + }; \ + config_t const level_##x##_dict = { \ + .name = "level " #x " with dict", \ + .cli_args = "-" #x, \ + .param_values = PARAM_VALUES(level_##x##_param_values), \ + .use_dictionary = 1, \ + }; + +#define PARAM_VALUES(pv) \ + { .data = pv, .size = sizeof(pv) / sizeof((pv)[0]) } + +#include "levels.h" + +#undef LEVEL +#undef FAST_LEVEL + +static config_t no_pledged_src_size = { + .name = "no source size", + .cli_args = "", + .param_values = PARAM_VALUES(level_0_param_values), + .no_pledged_src_size = 1, +}; + +static param_value_t const ldm_param_values[] = { + {.param = ZSTD_c_enableLongDistanceMatching, .value = 1}, +}; + +static config_t ldm = { + .name = "long distance mode", + .cli_args = "--long", + .param_values = PARAM_VALUES(ldm_param_values), +}; + +static param_value_t const mt_param_values[] = { + {.param = ZSTD_c_nbWorkers, .value = 2}, +}; + +static config_t mt = { + .name = "multithreaded", + .cli_args = "-T2", + .param_values = PARAM_VALUES(mt_param_values), +}; + +static param_value_t const mt_ldm_param_values[] = { + {.param = ZSTD_c_nbWorkers, .value = 2}, + {.param = ZSTD_c_enableLongDistanceMatching, .value = 1}, +}; + +static config_t mt_ldm = { + .name = "multithreaded long distance mode", + .cli_args = "-T2 --long", + .param_values = PARAM_VALUES(mt_ldm_param_values), +}; + +static param_value_t const small_wlog_param_values[] = { + {.param = ZSTD_c_windowLog, .value = 10}, +}; + +static config_t small_wlog = { + .name = "small window log", + .cli_args = "--zstd=wlog=10", + .param_values = PARAM_VALUES(small_wlog_param_values), +}; + +static param_value_t const small_hlog_param_values[] = { + {.param = ZSTD_c_hashLog, .value = 6}, + {.param = ZSTD_c_strategy, .value = (int)ZSTD_btopt}, +}; + +static config_t small_hlog = { + .name = "small hash log", + .cli_args = "--zstd=hlog=6,strat=7", + .param_values = PARAM_VALUES(small_hlog_param_values), +}; + +static param_value_t const small_clog_param_values[] = { + {.param = ZSTD_c_chainLog, .value = 6}, + {.param = ZSTD_c_strategy, .value = (int)ZSTD_btopt}, +}; + +static config_t small_clog = { + .name = "small chain log", + .cli_args = "--zstd=clog=6,strat=7", + .param_values = PARAM_VALUES(small_clog_param_values), +}; + +static param_value_t const explicit_params_param_values[] = { + {.param = ZSTD_c_checksumFlag, .value = 1}, + {.param = ZSTD_c_contentSizeFlag, .value = 0}, + {.param = ZSTD_c_dictIDFlag, .value = 0}, + {.param = ZSTD_c_strategy, .value = (int)ZSTD_greedy}, + {.param = ZSTD_c_windowLog, .value = 18}, + {.param = ZSTD_c_hashLog, .value = 21}, + {.param = ZSTD_c_chainLog, .value = 21}, + {.param = ZSTD_c_targetLength, .value = 100}, +}; + +static config_t explicit_params = { + .name = "explicit params", + .cli_args = "--no-check --no-dictID --zstd=strategy=3,wlog=18,hlog=21,clog=21,tlen=100", + .param_values = PARAM_VALUES(explicit_params_param_values), +}; + +static config_t const* g_configs[] = { + +#define FAST_LEVEL(x) &level_fast##x, &level_fast##x##_dict, +#define LEVEL(x) &level_##x, &level_##x##_dict, +#include "levels.h" +#undef LEVEL +#undef FAST_LEVEL + + &no_pledged_src_size, + &ldm, + &mt, + &mt_ldm, + &small_wlog, + &small_hlog, + &small_clog, + &explicit_params, + NULL, +}; + +config_t const* const* configs = g_configs; + +int config_skip_data(config_t const* config, data_t const* data) { + return config->use_dictionary && !data_has_dict(data); +} + +int config_get_level(config_t const* config) +{ + param_values_t const params = config->param_values; + size_t i; + for (i = 0; i < params.size; ++i) { + if (params.data[i].param == ZSTD_c_compressionLevel) + return (int)params.data[i].value; + } + return CONFIG_NO_LEVEL; +} + +ZSTD_parameters config_get_zstd_params( + config_t const* config, + uint64_t srcSize, + size_t dictSize) +{ + ZSTD_parameters zparams = {}; + param_values_t const params = config->param_values; + int level = config_get_level(config); + if (level == CONFIG_NO_LEVEL) + level = 3; + zparams = ZSTD_getParams( + level, + config->no_pledged_src_size ? ZSTD_CONTENTSIZE_UNKNOWN : srcSize, + dictSize); + for (size_t i = 0; i < params.size; ++i) { + unsigned const value = params.data[i].value; + switch (params.data[i].param) { + case ZSTD_c_contentSizeFlag: + zparams.fParams.contentSizeFlag = value; + break; + case ZSTD_c_checksumFlag: + zparams.fParams.checksumFlag = value; + break; + case ZSTD_c_dictIDFlag: + zparams.fParams.noDictIDFlag = !value; + break; + case ZSTD_c_windowLog: + zparams.cParams.windowLog = value; + break; + case ZSTD_c_chainLog: + zparams.cParams.chainLog = value; + break; + case ZSTD_c_hashLog: + zparams.cParams.hashLog = value; + break; + case ZSTD_c_searchLog: + zparams.cParams.searchLog = value; + break; + case ZSTD_c_minMatch: + zparams.cParams.minMatch = value; + break; + case ZSTD_c_targetLength: + zparams.cParams.targetLength = value; + break; + case ZSTD_c_strategy: + zparams.cParams.strategy = (ZSTD_strategy)value; + break; + default: + break; + } + } + return zparams; +} diff --git a/tests/regression/config.h b/tests/regression/config.h new file mode 100644 index 000000000000..3cd0308a0987 --- /dev/null +++ b/tests/regression/config.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef CONFIG_H +#define CONFIG_H + +#include <stddef.h> + +#define ZSTD_STATIC_LINKING_ONLY +#include <zstd.h> + +#include "data.h" + +typedef struct { + ZSTD_cParameter param; + int value; +} param_value_t; + +typedef struct { + size_t size; + param_value_t const* data; +} param_values_t; + +/** + * The config tells the compression method what options to use. + */ +typedef struct { + const char* name; /**< Identifies the config in the results table */ + /** + * Optional arguments to pass to the CLI. If not set, CLI-based methods + * will skip this config. + */ + char const* cli_args; + /** + * Parameters to pass to the advanced API. If the advanced API isn't used, + * the parameters will be derived from these. + */ + param_values_t param_values; + /** + * Boolean parameter that says if we should use a dictionary. If the data + * doesn't have a dictionary, this config is skipped. Defaults to no. + */ + int use_dictionary; + /** + * Boolean parameter that says if we should pass the pledged source size + * when the method allows it. Defaults to yes. + */ + int no_pledged_src_size; +} config_t; + +/** + * Returns true if the config should skip this data. + * For instance, if the config requires a dictionary but the data doesn't have + * one. + */ +int config_skip_data(config_t const* config, data_t const* data); + +#define CONFIG_NO_LEVEL (-ZSTD_TARGETLENGTH_MAX - 1) +/** + * Returns the compression level specified by the config, or CONFIG_NO_LEVEL if + * no level is specified. Note that 0 is a valid compression level, meaning + * default. + */ +int config_get_level(config_t const* config); + +/** + * Returns the compression parameters specified by the config. + */ +ZSTD_parameters config_get_zstd_params( + config_t const* config, + uint64_t srcSize, + size_t dictSize); + +/** + * The NULL-terminated list of configs. + */ +extern config_t const* const* configs; + +#endif diff --git a/tests/regression/data.c b/tests/regression/data.c new file mode 100644 index 000000000000..86e7687de5ee --- /dev/null +++ b/tests/regression/data.c @@ -0,0 +1,617 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "data.h" + +#include <assert.h> +#include <errno.h> +#include <stdio.h> +#include <string.h> + +#include <sys/stat.h> + +#include <curl/curl.h> + +#include "mem.h" +#include "util.h" +#define XXH_STATIC_LINKING_ONLY +#include "xxhash.h" + +/** + * Data objects + */ + +#define REGRESSION_RELEASE(x) \ + "https://github.com/facebook/zstd/releases/download/regression-data/" x + +data_t silesia = { + .name = "silesia", + .type = data_type_dir, + .data = + { + .url = REGRESSION_RELEASE("silesia.tar.zst"), + .xxhash64 = 0x48a199f92f93e977LL, + }, +}; + +data_t silesia_tar = { + .name = "silesia.tar", + .type = data_type_file, + .data = + { + .url = REGRESSION_RELEASE("silesia.tar.zst"), + .xxhash64 = 0x48a199f92f93e977LL, + }, +}; + +data_t github = { + .name = "github", + .type = data_type_dir, + .data = + { + .url = REGRESSION_RELEASE("github.tar.zst"), + .xxhash64 = 0xa9b1b44b020df292LL, + }, + .dict = + { + .url = REGRESSION_RELEASE("github.dict.zst"), + .xxhash64 = 0x1eddc6f737d3cb53LL, + + }, +}; + +static data_t* g_data[] = { + &silesia, + &silesia_tar, + &github, + NULL, +}; + +data_t const* const* data = (data_t const* const*)g_data; + +/** + * data helpers. + */ + +int data_has_dict(data_t const* data) { + return data->dict.url != NULL; +} + +/** + * data buffer helper functions (documented in header). + */ + +data_buffer_t data_buffer_create(size_t const capacity) { + data_buffer_t buffer = {}; + + buffer.data = (uint8_t*)malloc(capacity); + if (buffer.data == NULL) + return buffer; + buffer.capacity = capacity; + return buffer; +} + +data_buffer_t data_buffer_read(char const* filename) { + data_buffer_t buffer = {}; + + uint64_t const size = UTIL_getFileSize(filename); + if (size == UTIL_FILESIZE_UNKNOWN) { + fprintf(stderr, "unknown size for %s\n", filename); + return buffer; + } + + buffer.data = (uint8_t*)malloc(size); + if (buffer.data == NULL) { + fprintf(stderr, "malloc failed\n"); + return buffer; + } + buffer.capacity = size; + + FILE* file = fopen(filename, "rb"); + if (file == NULL) { + fprintf(stderr, "file null\n"); + goto err; + } + buffer.size = fread(buffer.data, 1, buffer.capacity, file); + fclose(file); + if (buffer.size != buffer.capacity) { + fprintf(stderr, "read %zu != %zu\n", buffer.size, buffer.capacity); + goto err; + } + + return buffer; +err: + free(buffer.data); + memset(&buffer, 0, sizeof(buffer)); + return buffer; +} + +data_buffer_t data_buffer_get_data(data_t const* data) { + data_buffer_t const kEmptyBuffer = {}; + + if (data->type != data_type_file) + return kEmptyBuffer; + + return data_buffer_read(data->data.path); +} + +data_buffer_t data_buffer_get_dict(data_t const* data) { + data_buffer_t const kEmptyBuffer = {}; + + if (!data_has_dict(data)) + return kEmptyBuffer; + + return data_buffer_read(data->dict.path); +} + +int data_buffer_compare(data_buffer_t buffer1, data_buffer_t buffer2) { + size_t const size = + buffer1.size < buffer2.size ? buffer1.size : buffer2.size; + int const cmp = memcmp(buffer1.data, buffer2.data, size); + if (cmp != 0) + return cmp; + if (buffer1.size < buffer2.size) + return -1; + if (buffer1.size == buffer2.size) + return 0; + assert(buffer1.size > buffer2.size); + return 1; +} + +void data_buffer_free(data_buffer_t buffer) { + free(buffer.data); +} + +/** + * data filenames helpers. + */ + +data_filenames_t data_filenames_get(data_t const* data) { + data_filenames_t filenames = {.buffer = NULL, .size = 0}; + char const* path = data->data.path; + + filenames.filenames = UTIL_createFileList( + &path, + 1, + &filenames.buffer, + &filenames.size, + /* followLinks */ 0); + return filenames; +} + +void data_filenames_free(data_filenames_t filenames) { + UTIL_freeFileList(filenames.filenames, filenames.buffer); +} + +/** + * data buffers helpers. + */ + +data_buffers_t data_buffers_get(data_t const* data) { + data_buffers_t buffers = {.size = 0}; + data_filenames_t filenames = data_filenames_get(data); + if (filenames.size == 0) + return buffers; + + data_buffer_t* buffersPtr = + (data_buffer_t*)malloc(filenames.size * sizeof(data_buffer_t)); + if (buffersPtr == NULL) + return buffers; + buffers.buffers = (data_buffer_t const*)buffersPtr; + buffers.size = filenames.size; + + for (size_t i = 0; i < filenames.size; ++i) { + buffersPtr[i] = data_buffer_read(filenames.filenames[i]); + if (buffersPtr[i].data == NULL) { + data_buffers_t const kEmptyBuffer = {}; + data_buffers_free(buffers); + return kEmptyBuffer; + } + } + + return buffers; +} + +/** + * Frees the data buffers. + */ +void data_buffers_free(data_buffers_t buffers) { + free((data_buffer_t*)buffers.buffers); +} + +/** + * Initialization and download functions. + */ + +static char* g_data_dir = NULL; + +/* mkdir -p */ +static int ensure_directory_exists(char const* indir) { + char* const dir = strdup(indir); + char* end = dir; + int ret = 0; + if (dir == NULL) { + ret = EINVAL; + goto out; + } + do { + /* Find the next directory level. */ + for (++end; *end != '\0' && *end != '/'; ++end) + ; + /* End the string there, make the directory, and restore the string. */ + char const save = *end; + *end = '\0'; + int const isdir = UTIL_isDirectory(dir); + ret = mkdir(dir, S_IRWXU); + *end = save; + /* Its okay if the directory already exists. */ + if (ret == 0 || (errno == EEXIST && isdir)) + continue; + ret = errno; + fprintf(stderr, "mkdir() failed\n"); + goto out; + } while (*end != '\0'); + + ret = 0; +out: + free(dir); + return ret; +} + +/** Concatenate 3 strings into a new buffer. */ +static char* cat3(char const* str1, char const* str2, char const* str3) { + size_t const size1 = strlen(str1); + size_t const size2 = strlen(str2); + size_t const size3 = str3 == NULL ? 0 : strlen(str3); + size_t const size = size1 + size2 + size3 + 1; + char* const dst = (char*)malloc(size); + if (dst == NULL) + return NULL; + strcpy(dst, str1); + strcpy(dst + size1, str2); + if (str3 != NULL) + strcpy(dst + size1 + size2, str3); + assert(strlen(dst) == size1 + size2 + size3); + return dst; +} + +static char* cat2(char const* str1, char const* str2) { + return cat3(str1, str2, NULL); +} + +/** + * State needed by the curl callback. + * It takes data from curl, hashes it, and writes it to the file. + */ +typedef struct { + FILE* file; + XXH64_state_t xxhash64; + int error; +} curl_data_t; + +/** Create the curl state. */ +static curl_data_t curl_data_create( + data_resource_t const* resource, + data_type_t type) { + curl_data_t cdata = {}; + + XXH64_reset(&cdata.xxhash64, 0); + + assert(UTIL_isDirectory(g_data_dir)); + + if (type == data_type_file) { + /* Decompress the resource and store to the path. */ + char* cmd = cat3("zstd -dqfo '", resource->path, "'"); + if (cmd == NULL) { + cdata.error = ENOMEM; + return cdata; + } + cdata.file = popen(cmd, "w"); + free(cmd); + } else { + /* Decompress and extract the resource to the cache directory. */ + char* cmd = cat3("zstd -dc | tar -x -C '", g_data_dir, "'"); + if (cmd == NULL) { + cdata.error = ENOMEM; + return cdata; + } + cdata.file = popen(cmd, "w"); + free(cmd); + } + if (cdata.file == NULL) { + cdata.error = errno; + } + + return cdata; +} + +/** Free the curl state. */ +static int curl_data_free(curl_data_t cdata) { + return pclose(cdata.file); +} + +/** curl callback. Updates the hash, and writes to the file. */ +static size_t curl_write(void* data, size_t size, size_t count, void* ptr) { + curl_data_t* cdata = (curl_data_t*)ptr; + size_t const written = fwrite(data, size, count, cdata->file); + XXH64_update(&cdata->xxhash64, data, written * size); + return written; +} + +static int curl_download_resource( + CURL* curl, + data_resource_t const* resource, + data_type_t type) { + curl_data_t cdata; + /* Download the data. */ + if (curl_easy_setopt(curl, CURLOPT_URL, resource->url) != 0) + return EINVAL; + if (curl_easy_setopt(curl, CURLOPT_WRITEDATA, &cdata) != 0) + return EINVAL; + cdata = curl_data_create(resource, type); + if (cdata.error != 0) + return cdata.error; + int const curl_err = curl_easy_perform(curl); + int const close_err = curl_data_free(cdata); + if (curl_err) { + fprintf( + stderr, + "downloading '%s' for '%s' failed\n", + resource->url, + resource->path); + return EIO; + } + if (close_err) { + fprintf(stderr, "writing data to '%s' failed\n", resource->path); + return EIO; + } + /* check that the file exists. */ + if (type == data_type_file && !UTIL_isRegularFile(resource->path)) { + fprintf(stderr, "output file '%s' does not exist\n", resource->path); + return EIO; + } + if (type == data_type_dir && !UTIL_isDirectory(resource->path)) { + fprintf( + stderr, "output directory '%s' does not exist\n", resource->path); + return EIO; + } + /* Check that the hash matches. */ + if (XXH64_digest(&cdata.xxhash64) != resource->xxhash64) { + fprintf( + stderr, + "checksum does not match: 0x%llxLL != 0x%llxLL\n", + (unsigned long long)XXH64_digest(&cdata.xxhash64), + (unsigned long long)resource->xxhash64); + return EINVAL; + } + + return 0; +} + +/** Download a single data object. */ +static int curl_download_datum(CURL* curl, data_t const* data) { + int ret; + ret = curl_download_resource(curl, &data->data, data->type); + if (ret != 0) + return ret; + if (data_has_dict(data)) { + ret = curl_download_resource(curl, &data->dict, data_type_file); + if (ret != 0) + return ret; + } + return ret; +} + +/** Download all the data. */ +static int curl_download_data(data_t const* const* data) { + if (curl_global_init(CURL_GLOBAL_ALL) != 0) + return EFAULT; + + curl_data_t cdata = {}; + CURL* curl = curl_easy_init(); + int err = EFAULT; + + if (curl == NULL) + return EFAULT; + + if (curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L) != 0) + goto out; + if (curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L) != 0) + goto out; + if (curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write) != 0) + goto out; + + assert(data != NULL); + for (; *data != NULL; ++data) { + if (curl_download_datum(curl, *data) != 0) + goto out; + } + + err = 0; +out: + curl_easy_cleanup(curl); + curl_global_cleanup(); + return err; +} + +/** Fill the path member variable of the data objects. */ +static int data_create_paths(data_t* const* data, char const* dir) { + size_t const dirlen = strlen(dir); + assert(data != NULL); + for (; *data != NULL; ++data) { + data_t* const datum = *data; + datum->data.path = cat3(dir, "/", datum->name); + if (datum->data.path == NULL) + return ENOMEM; + if (data_has_dict(datum)) { + datum->dict.path = cat2(datum->data.path, ".dict"); + if (datum->dict.path == NULL) + return ENOMEM; + } + } + return 0; +} + +/** Free the path member variable of the data objects. */ +static void data_free_paths(data_t* const* data) { + assert(data != NULL); + for (; *data != NULL; ++data) { + data_t* datum = *data; + free((void*)datum->data.path); + free((void*)datum->dict.path); + datum->data.path = NULL; + datum->dict.path = NULL; + } +} + +static char const kStampName[] = "STAMP"; + +static void xxh_update_le(XXH64_state_t* state, uint64_t data) { + if (!MEM_isLittleEndian()) + data = MEM_swap64(data); + XXH64_update(state, &data, sizeof(data)); +} + +/** Hash the data to create the stamp. */ +static uint64_t stamp_hash(data_t const* const* data) { + XXH64_state_t state; + + XXH64_reset(&state, 0); + assert(data != NULL); + for (; *data != NULL; ++data) { + data_t const* datum = *data; + /* We don't care about the URL that we fetch from. */ + /* The path is derived from the name. */ + XXH64_update(&state, datum->name, strlen(datum->name)); + xxh_update_le(&state, datum->data.xxhash64); + xxh_update_le(&state, datum->dict.xxhash64); + xxh_update_le(&state, datum->type); + } + return XXH64_digest(&state); +} + +/** Check if the stamp matches the stamp in the cache directory. */ +static int stamp_check(char const* dir, data_t const* const* data) { + char* stamp = cat3(dir, "/", kStampName); + uint64_t const expected = stamp_hash(data); + XXH64_canonical_t actual; + FILE* stampfile = NULL; + int matches = 0; + + if (stamp == NULL) + goto out; + if (!UTIL_isRegularFile(stamp)) { + fprintf(stderr, "stamp does not exist: recreating the data cache\n"); + goto out; + } + + stampfile = fopen(stamp, "rb"); + if (stampfile == NULL) { + fprintf(stderr, "could not open stamp: recreating the data cache\n"); + goto out; + } + + size_t b; + if ((b = fread(&actual, sizeof(actual), 1, stampfile)) != 1) { + fprintf(stderr, "invalid stamp: recreating the data cache\n"); + goto out; + } + + matches = (expected == XXH64_hashFromCanonical(&actual)); + if (matches) + fprintf(stderr, "stamp matches: reusing the cached data\n"); + else + fprintf(stderr, "stamp does not match: recreating the data cache\n"); + +out: + free(stamp); + if (stampfile != NULL) + fclose(stampfile); + return matches; +} + +/** On success write a new stamp, on failure delete the old stamp. */ +static int +stamp_write(char const* dir, data_t const* const* data, int const data_err) { + char* stamp = cat3(dir, "/", kStampName); + FILE* stampfile = NULL; + int err = EIO; + + if (stamp == NULL) + return ENOMEM; + + if (data_err != 0) { + err = data_err; + goto out; + } + XXH64_canonical_t hash; + + XXH64_canonicalFromHash(&hash, stamp_hash(data)); + + stampfile = fopen(stamp, "wb"); + if (stampfile == NULL) + goto out; + if (fwrite(&hash, sizeof(hash), 1, stampfile) != 1) + goto out; + err = 0; + fprintf(stderr, "stamped new data cache\n"); +out: + if (err != 0) + /* Ignore errors. */ + unlink(stamp); + free(stamp); + if (stampfile != NULL) + fclose(stampfile); + return err; +} + +int data_init(char const* dir) { + int err; + + if (dir == NULL) + return EINVAL; + + /* This must be first to simplify logic. */ + err = ensure_directory_exists(dir); + if (err != 0) + return err; + + /* Save the cache directory. */ + g_data_dir = strdup(dir); + if (g_data_dir == NULL) + return ENOMEM; + + err = data_create_paths(g_data, dir); + if (err != 0) + return err; + + /* If the stamp matches then we are good to go. + * This must be called before any modifications to the data cache. + * After this point, we MUST call stamp_write() to update the STAMP, + * since we've updated the data cache. + */ + if (stamp_check(dir, data)) + return 0; + + err = curl_download_data(data); + if (err != 0) + goto out; + +out: + /* This must be last, since it must know if data_init() succeeded. */ + stamp_write(dir, data, err); + return err; +} + +void data_finish(void) { + data_free_paths(g_data); + free(g_data_dir); + g_data_dir = NULL; +} diff --git a/tests/regression/data.h b/tests/regression/data.h new file mode 100644 index 000000000000..717fe1294c14 --- /dev/null +++ b/tests/regression/data.h @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef DATA_H +#define DATA_H + +#include <stddef.h> +#include <stdint.h> + +typedef enum { + data_type_file = 1, /**< This data is a file. *.zst */ + data_type_dir = 2, /**< This data is a directory. *.tar.zst */ +} data_type_t; + +typedef struct { + char const* url; /**< Where to get this resource. */ + uint64_t xxhash64; /**< Hash of the url contents. */ + char const* path; /**< The path of the unpacked resource (derived). */ +} data_resource_t; + +typedef struct { + data_resource_t data; + data_resource_t dict; + data_type_t type; /**< The type of the data. */ + char const* name; /**< The logical name of the data (no extension). */ +} data_t; + +/** + * The NULL-terminated list of data objects. + */ +extern data_t const* const* data; + + +int data_has_dict(data_t const* data); + +/** + * Initializes the data module and downloads the data necessary. + * Caches the downloads in dir. We add a stamp file in the directory after + * a successful download. If a stamp file already exists, and matches our + * current data stamp, we will use the cached data without downloading. + * + * @param dir The directory to cache the downloaded data into. + * + * @returns 0 on success. + */ +int data_init(char const* dir); + +/** + * Must be called at exit to free resources allocated by data_init(). + */ +void data_finish(void); + +typedef struct { + uint8_t* data; + size_t size; + size_t capacity; +} data_buffer_t; + +/** + * Read the file that data points to into a buffer. + * NOTE: data must be a file, not a directory. + * + * @returns The buffer, which is NULL on failure. + */ +data_buffer_t data_buffer_get_data(data_t const* data); + +/** + * Read the dictionary that the data points to into a buffer. + * + * @returns The buffer, which is NULL on failure. + */ +data_buffer_t data_buffer_get_dict(data_t const* data); + +/** + * Read the contents of filename into a buffer. + * + * @returns The buffer, which is NULL on failure. + */ +data_buffer_t data_buffer_read(char const* filename); + +/** + * Create a buffer with the specified capacity. + * + * @returns The buffer, which is NULL on failure. + */ +data_buffer_t data_buffer_create(size_t capacity); + +/** + * Calls memcmp() on the contents [0, size) of both buffers. + */ +int data_buffer_compare(data_buffer_t buffer1, data_buffer_t buffer2); + +/** + * Frees an allocated buffer. + */ +void data_buffer_free(data_buffer_t buffer); + +typedef struct { + char* buffer; + char const** filenames; + unsigned size; +} data_filenames_t; + +/** + * Get a recursive list of filenames in the data object. If it is a file, it + * will only contain one entry. If it is a directory, it will recursively walk + * the directory. + * + * @returns The list of filenames, which has size 0 and NULL pointers on error. + */ +data_filenames_t data_filenames_get(data_t const* data); + +/** + * Frees the filenames table. + */ +void data_filenames_free(data_filenames_t filenames); + +typedef struct { + data_buffer_t const* buffers; + size_t size; +} data_buffers_t; + +/** + * @returns a list of buffers for every file in data. It is zero sized on error. + */ +data_buffers_t data_buffers_get(data_t const* data); + +/** + * Frees the data buffers. + */ +void data_buffers_free(data_buffers_t buffers); + +#endif diff --git a/tests/regression/levels.h b/tests/regression/levels.h new file mode 100644 index 000000000000..f96689075126 --- /dev/null +++ b/tests/regression/levels.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef LEVEL +# error LEVEL(x) must be defined +#endif +#ifndef FAST_LEVEL +# error FAST_LEVEL(x) must be defined +#endif + +/** + * The levels are chosen to trigger every strategy in every source size, + * as well as some fast levels and the default level. + * If you change the compression levels, you should probably update these. + */ + +FAST_LEVEL(5) + +FAST_LEVEL(3) + +FAST_LEVEL(1) +LEVEL(0) +LEVEL(1) + +LEVEL(3) +LEVEL(4) +LEVEL(5) +LEVEL(6) +LEVEL(7) + +LEVEL(9) + +LEVEL(13) + +LEVEL(16) + +LEVEL(19) diff --git a/tests/regression/method.c b/tests/regression/method.c new file mode 100644 index 000000000000..51a0fb9d14f0 --- /dev/null +++ b/tests/regression/method.c @@ -0,0 +1,566 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "method.h" + +#include <stdio.h> +#include <stdlib.h> + +#define ZSTD_STATIC_LINKING_ONLY +#include <zstd.h> + +#define MIN(x, y) ((x) < (y) ? (x) : (y)) + +static char const* g_zstdcli = NULL; + +void method_set_zstdcli(char const* zstdcli) { + g_zstdcli = zstdcli; +} + +/** + * Macro to get a pointer of type, given ptr, which is a member variable with + * the given name, member. + * + * method_state_t* base = ...; + * buffer_state_t* state = container_of(base, buffer_state_t, base); + */ +#define container_of(ptr, type, member) \ + ((type*)(ptr == NULL ? NULL : (char*)(ptr)-offsetof(type, member))) + +/** State to reuse the same buffers between compression calls. */ +typedef struct { + method_state_t base; + data_buffers_t inputs; /**< The input buffer for each file. */ + data_buffer_t dictionary; /**< The dictionary. */ + data_buffer_t compressed; /**< The compressed data buffer. */ + data_buffer_t decompressed; /**< The decompressed data buffer. */ +} buffer_state_t; + +static size_t buffers_max_size(data_buffers_t buffers) { + size_t max = 0; + for (size_t i = 0; i < buffers.size; ++i) { + if (buffers.buffers[i].size > max) + max = buffers.buffers[i].size; + } + return max; +} + +static method_state_t* buffer_state_create(data_t const* data) { + buffer_state_t* state = (buffer_state_t*)calloc(1, sizeof(buffer_state_t)); + if (state == NULL) + return NULL; + state->base.data = data; + state->inputs = data_buffers_get(data); + state->dictionary = data_buffer_get_dict(data); + size_t const max_size = buffers_max_size(state->inputs); + state->compressed = data_buffer_create(ZSTD_compressBound(max_size)); + state->decompressed = data_buffer_create(max_size); + return &state->base; +} + +static void buffer_state_destroy(method_state_t* base) { + if (base == NULL) + return; + buffer_state_t* state = container_of(base, buffer_state_t, base); + free(state); +} + +static int buffer_state_bad( + buffer_state_t const* state, + config_t const* config) { + if (state == NULL) { + fprintf(stderr, "buffer_state_t is NULL\n"); + return 1; + } + if (state->inputs.size == 0 || state->compressed.data == NULL || + state->decompressed.data == NULL) { + fprintf(stderr, "buffer state allocation failure\n"); + return 1; + } + if (config->use_dictionary && state->dictionary.data == NULL) { + fprintf(stderr, "dictionary loading failed\n"); + return 1; + } + return 0; +} + +static result_t simple_compress(method_state_t* base, config_t const* config) { + buffer_state_t* state = container_of(base, buffer_state_t, base); + + if (buffer_state_bad(state, config)) + return result_error(result_error_system_error); + + /* Keep the tests short by skipping directories, since behavior shouldn't + * change. + */ + if (base->data->type != data_type_file) + return result_error(result_error_skip); + + if (config->use_dictionary || config->no_pledged_src_size) + return result_error(result_error_skip); + + /* If the config doesn't specify a level, skip. */ + int const level = config_get_level(config); + if (level == CONFIG_NO_LEVEL) + return result_error(result_error_skip); + + data_buffer_t const input = state->inputs.buffers[0]; + + /* Compress, decompress, and check the result. */ + state->compressed.size = ZSTD_compress( + state->compressed.data, + state->compressed.capacity, + input.data, + input.size, + level); + if (ZSTD_isError(state->compressed.size)) + return result_error(result_error_compression_error); + + state->decompressed.size = ZSTD_decompress( + state->decompressed.data, + state->decompressed.capacity, + state->compressed.data, + state->compressed.size); + if (ZSTD_isError(state->decompressed.size)) + return result_error(result_error_decompression_error); + if (data_buffer_compare(input, state->decompressed)) + return result_error(result_error_round_trip_error); + + result_data_t data; + data.total_size = state->compressed.size; + return result_data(data); +} + +static result_t compress_cctx_compress( + method_state_t* base, + config_t const* config) { + buffer_state_t* state = container_of(base, buffer_state_t, base); + + if (buffer_state_bad(state, config)) + return result_error(result_error_system_error); + + if (config->no_pledged_src_size) + return result_error(result_error_skip); + + if (base->data->type != data_type_dir) + return result_error(result_error_skip); + + int const level = config_get_level(config); + + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + if (cctx == NULL || dctx == NULL) { + fprintf(stderr, "context creation failed\n"); + return result_error(result_error_system_error); + } + + result_t result; + result_data_t data = {.total_size = 0}; + for (size_t i = 0; i < state->inputs.size; ++i) { + data_buffer_t const input = state->inputs.buffers[i]; + ZSTD_parameters const params = + config_get_zstd_params(config, input.size, state->dictionary.size); + + if (level == CONFIG_NO_LEVEL) + state->compressed.size = ZSTD_compress_advanced( + cctx, + state->compressed.data, + state->compressed.capacity, + input.data, + input.size, + state->dictionary.data, + state->dictionary.size, + params); + else if (config->use_dictionary) + state->compressed.size = ZSTD_compress_usingDict( + cctx, + state->compressed.data, + state->compressed.capacity, + input.data, + input.size, + state->dictionary.data, + state->dictionary.size, + level); + else + state->compressed.size = ZSTD_compressCCtx( + cctx, + state->compressed.data, + state->compressed.capacity, + input.data, + input.size, + level); + + if (ZSTD_isError(state->compressed.size)) { + result = result_error(result_error_compression_error); + goto out; + } + + if (config->use_dictionary) + state->decompressed.size = ZSTD_decompress_usingDict( + dctx, + state->decompressed.data, + state->decompressed.capacity, + state->compressed.data, + state->compressed.size, + state->dictionary.data, + state->dictionary.size); + else + state->decompressed.size = ZSTD_decompressDCtx( + dctx, + state->decompressed.data, + state->decompressed.capacity, + state->compressed.data, + state->compressed.size); + if (ZSTD_isError(state->decompressed.size)) { + result = result_error(result_error_decompression_error); + goto out; + } + if (data_buffer_compare(input, state->decompressed)) { + result = result_error(result_error_round_trip_error); + goto out; + } + + data.total_size += state->compressed.size; + } + + result = result_data(data); +out: + ZSTD_freeCCtx(cctx); + ZSTD_freeDCtx(dctx); + return result; +} + +/** Generic state creation function. */ +static method_state_t* method_state_create(data_t const* data) { + method_state_t* state = (method_state_t*)malloc(sizeof(method_state_t)); + if (state == NULL) + return NULL; + state->data = data; + return state; +} + +static void method_state_destroy(method_state_t* state) { + free(state); +} + +static result_t cli_compress(method_state_t* state, config_t const* config) { + if (config->cli_args == NULL) + return result_error(result_error_skip); + + /* We don't support no pledged source size with directories. Too slow. */ + if (state->data->type == data_type_dir && config->no_pledged_src_size) + return result_error(result_error_skip); + + if (g_zstdcli == NULL) + return result_error(result_error_system_error); + + /* '<zstd>' -cqr <args> [-D '<dict>'] '<file/dir>' */ + char cmd[1024]; + size_t const cmd_size = snprintf( + cmd, + sizeof(cmd), + "'%s' -cqr %s %s%s%s %s '%s'", + g_zstdcli, + config->cli_args, + config->use_dictionary ? "-D '" : "", + config->use_dictionary ? state->data->dict.path : "", + config->use_dictionary ? "'" : "", + config->no_pledged_src_size ? "<" : "", + state->data->data.path); + if (cmd_size >= sizeof(cmd)) { + fprintf(stderr, "command too large: %s\n", cmd); + return result_error(result_error_system_error); + } + FILE* zstd = popen(cmd, "r"); + if (zstd == NULL) { + fprintf(stderr, "failed to popen command: %s\n", cmd); + return result_error(result_error_system_error); + } + + char out[4096]; + size_t total_size = 0; + while (1) { + size_t const size = fread(out, 1, sizeof(out), zstd); + total_size += size; + if (size != sizeof(out)) + break; + } + if (ferror(zstd) || pclose(zstd) != 0) { + fprintf(stderr, "zstd failed with command: %s\n", cmd); + return result_error(result_error_compression_error); + } + + result_data_t const data = {.total_size = total_size}; + return result_data(data); +} + +static int advanced_config( + ZSTD_CCtx* cctx, + buffer_state_t* state, + config_t const* config) { + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + for (size_t p = 0; p < config->param_values.size; ++p) { + param_value_t const pv = config->param_values.data[p]; + if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, pv.param, pv.value))) { + return 1; + } + } + if (config->use_dictionary) { + if (ZSTD_isError(ZSTD_CCtx_loadDictionary( + cctx, state->dictionary.data, state->dictionary.size))) { + return 1; + } + } + return 0; +} + +static result_t advanced_one_pass_compress_output_adjustment( + method_state_t* base, + config_t const* config, + size_t const subtract) { + buffer_state_t* state = container_of(base, buffer_state_t, base); + + if (buffer_state_bad(state, config)) + return result_error(result_error_system_error); + + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + result_t result; + + if (!cctx || advanced_config(cctx, state, config)) { + result = result_error(result_error_compression_error); + goto out; + } + + result_data_t data = {.total_size = 0}; + for (size_t i = 0; i < state->inputs.size; ++i) { + data_buffer_t const input = state->inputs.buffers[i]; + + if (!config->no_pledged_src_size) { + if (ZSTD_isError(ZSTD_CCtx_setPledgedSrcSize(cctx, input.size))) { + result = result_error(result_error_compression_error); + goto out; + } + } + size_t const size = ZSTD_compress2( + cctx, + state->compressed.data, + ZSTD_compressBound(input.size) - subtract, + input.data, + input.size); + if (ZSTD_isError(size)) { + result = result_error(result_error_compression_error); + goto out; + } + data.total_size += size; + } + + result = result_data(data); +out: + ZSTD_freeCCtx(cctx); + return result; +} + +static result_t advanced_one_pass_compress( + method_state_t* base, + config_t const* config) { + return advanced_one_pass_compress_output_adjustment(base, config, 0); +} + +static result_t advanced_one_pass_compress_small_output( + method_state_t* base, + config_t const* config) { + return advanced_one_pass_compress_output_adjustment(base, config, 1); +} + +static result_t advanced_streaming_compress( + method_state_t* base, + config_t const* config) { + buffer_state_t* state = container_of(base, buffer_state_t, base); + + if (buffer_state_bad(state, config)) + return result_error(result_error_system_error); + + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + result_t result; + + if (!cctx || advanced_config(cctx, state, config)) { + result = result_error(result_error_compression_error); + goto out; + } + + result_data_t data = {.total_size = 0}; + for (size_t i = 0; i < state->inputs.size; ++i) { + data_buffer_t input = state->inputs.buffers[i]; + + if (!config->no_pledged_src_size) { + if (ZSTD_isError(ZSTD_CCtx_setPledgedSrcSize(cctx, input.size))) { + result = result_error(result_error_compression_error); + goto out; + } + } + + while (input.size > 0) { + ZSTD_inBuffer in = {input.data, MIN(input.size, 4096)}; + input.data += in.size; + input.size -= in.size; + ZSTD_EndDirective const op = + input.size > 0 ? ZSTD_e_continue : ZSTD_e_end; + size_t ret = 0; + while (in.pos < in.size || (op == ZSTD_e_end && ret != 0)) { + ZSTD_outBuffer out = {state->compressed.data, + MIN(state->compressed.capacity, 1024)}; + ret = ZSTD_compressStream2(cctx, &out, &in, op); + if (ZSTD_isError(ret)) { + result = result_error(result_error_compression_error); + goto out; + } + data.total_size += out.pos; + } + } + } + + result = result_data(data); +out: + ZSTD_freeCCtx(cctx); + return result; +} + +static result_t old_streaming_compress( + method_state_t* base, + config_t const* config) { + buffer_state_t* state = container_of(base, buffer_state_t, base); + + if (buffer_state_bad(state, config)) + return result_error(result_error_system_error); + + int const level = config_get_level(config); + if (level == CONFIG_NO_LEVEL) + return result_error(result_error_skip); + + ZSTD_CStream* zcs = ZSTD_createCStream(); + result_t result; + if (zcs == NULL) { + result = result_error(result_error_compression_error); + goto out; + } + size_t zret; + if (config->use_dictionary) { + zret = ZSTD_initCStream_usingDict( + zcs, state->dictionary.data, state->dictionary.size, level); + } else { + zret = ZSTD_initCStream(zcs, level); + } + if (ZSTD_isError(zret)) { + result = result_error(result_error_compression_error); + goto out; + } + + result_data_t data = {.total_size = 0}; + for (size_t i = 0; i < state->inputs.size; ++i) { + data_buffer_t input = state->inputs.buffers[i]; + zret = ZSTD_resetCStream( + zcs, + config->no_pledged_src_size ? ZSTD_CONTENTSIZE_UNKNOWN + : input.size); + if (ZSTD_isError(zret)) { + result = result_error(result_error_compression_error); + goto out; + } + + while (input.size > 0) { + ZSTD_inBuffer in = {input.data, MIN(input.size, 4096)}; + input.data += in.size; + input.size -= in.size; + ZSTD_EndDirective const op = + input.size > 0 ? ZSTD_e_continue : ZSTD_e_end; + zret = 0; + while (in.pos < in.size || (op == ZSTD_e_end && zret != 0)) { + ZSTD_outBuffer out = {state->compressed.data, + MIN(state->compressed.capacity, 1024)}; + if (op == ZSTD_e_continue || in.pos < in.size) + zret = ZSTD_compressStream(zcs, &out, &in); + else + zret = ZSTD_endStream(zcs, &out); + if (ZSTD_isError(zret)) { + result = result_error(result_error_compression_error); + goto out; + } + data.total_size += out.pos; + } + } + } + + result = result_data(data); +out: + ZSTD_freeCStream(zcs); + return result; +} + +method_t const simple = { + .name = "compress simple", + .create = buffer_state_create, + .compress = simple_compress, + .destroy = buffer_state_destroy, +}; + +method_t const compress_cctx = { + .name = "compress cctx", + .create = buffer_state_create, + .compress = compress_cctx_compress, + .destroy = buffer_state_destroy, +}; + +method_t const advanced_one_pass = { + .name = "advanced one pass", + .create = buffer_state_create, + .compress = advanced_one_pass_compress, + .destroy = buffer_state_destroy, +}; + +method_t const advanced_one_pass_small_out = { + .name = "advanced one pass small out", + .create = buffer_state_create, + .compress = advanced_one_pass_compress, + .destroy = buffer_state_destroy, +}; + +method_t const advanced_streaming = { + .name = "advanced streaming", + .create = buffer_state_create, + .compress = advanced_streaming_compress, + .destroy = buffer_state_destroy, +}; + +method_t const old_streaming = { + .name = "old streaming", + .create = buffer_state_create, + .compress = old_streaming_compress, + .destroy = buffer_state_destroy, +}; + +method_t const cli = { + .name = "zstdcli", + .create = method_state_create, + .compress = cli_compress, + .destroy = method_state_destroy, +}; + +static method_t const* g_methods[] = { + &simple, + &compress_cctx, + &cli, + &advanced_one_pass, + &advanced_one_pass_small_out, + &advanced_streaming, + &old_streaming, + NULL, +}; + +method_t const* const* methods = g_methods; diff --git a/tests/regression/method.h b/tests/regression/method.h new file mode 100644 index 000000000000..d70b776b1b90 --- /dev/null +++ b/tests/regression/method.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef METHOD_H +#define METHOD_H + +#include <stddef.h> + +#include "data.h" +#include "config.h" +#include "result.h" + +/** + * The base class for state that methods keep. + * All derived method state classes must have a member of this type. + */ +typedef struct { + data_t const* data; +} method_state_t; + +/** + * A method that compresses the data using config. + */ +typedef struct { + char const* name; /**< The identifier for this method in the results. */ + /** + * Creates a state that must contain a member variable of method_state_t, + * and returns a pointer to that member variable. + * + * This method can be used to do expensive work that only depends on the + * data, like loading the data file into a buffer. + */ + method_state_t* (*create)(data_t const* data); + /** + * Compresses the data in the state using the given config. + * + * @param state A pointer to the state returned by create(). + * + * @returns The total compressed size on success, or an error code. + */ + result_t (*compress)(method_state_t* state, config_t const* config); + /** + * Frees the state. + */ + void (*destroy)(method_state_t* state); +} method_t; + +/** + * Set the zstd cli path. Must be called before any methods are used. + */ +void method_set_zstdcli(char const* zstdcli); + +/** + * A NULL-terminated list of methods. + */ +extern method_t const* const* methods; + +#endif diff --git a/tests/regression/result.c b/tests/regression/result.c new file mode 100644 index 000000000000..31439b08cde7 --- /dev/null +++ b/tests/regression/result.c @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "result.h" + +char const* result_get_error_string(result_t result) { + switch (result_get_error(result)) { + case result_error_ok: + return "okay"; + case result_error_skip: + return "skip"; + case result_error_system_error: + return "system error"; + case result_error_compression_error: + return "compression error"; + case result_error_decompression_error: + return "decompression error"; + case result_error_round_trip_error: + return "round trip error"; + } +} diff --git a/tests/regression/result.h b/tests/regression/result.h new file mode 100644 index 000000000000..8c80cf85a966 --- /dev/null +++ b/tests/regression/result.h @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef RESULT_H +#define RESULT_H + +#include <stddef.h> + +/** + * The error type enum. + */ +typedef enum { + result_error_ok, /**< No error. */ + result_error_skip, /**< This method was skipped. */ + result_error_system_error, /**< Some internal error happened. */ + result_error_compression_error, /**< Compression failed. */ + result_error_decompression_error, /**< Decompression failed. */ + result_error_round_trip_error, /**< Data failed to round trip. */ +} result_error_t; + +/** + * The success type. + */ +typedef struct { + size_t total_size; /**< The total compressed size. */ +} result_data_t; + +/** + * The result type. + * Do not access the member variables directory, use the helper functions. + */ +typedef struct { + result_error_t internal_error; + result_data_t internal_data; +} result_t; + +/** + * Create a result of the error type. + */ +static result_t result_error(result_error_t error); +/** + * Create a result of the success type. + */ +static result_t result_data(result_data_t data); + +/** + * Check if the result is an error or skip. + */ +static int result_is_error(result_t result); +/** + * Check if the result error is skip. + */ +static int result_is_skip(result_t result); +/** + * Get the result error or okay. + */ +static result_error_t result_get_error(result_t result); +/** + * Get the result data. The result MUST be checked with result_is_error() first. + */ +static result_data_t result_get_data(result_t result); + +static result_t result_error(result_error_t error) { + result_t result = { + .internal_error = error, + }; + return result; +} + +static result_t result_data(result_data_t data) { + result_t result = { + .internal_error = result_error_ok, + .internal_data = data, + }; + return result; +} + +static int result_is_error(result_t result) { + return result_get_error(result) != result_error_ok; +} + +static int result_is_skip(result_t result) { + return result_get_error(result) == result_error_skip; +} + +static result_error_t result_get_error(result_t result) { + return result.internal_error; +} + +char const* result_get_error_string(result_t result); + +static result_data_t result_get_data(result_t result) { + return result.internal_data; +} + +#endif diff --git a/tests/regression/results.csv b/tests/regression/results.csv new file mode 100644 index 000000000000..a62178d6ed0c --- /dev/null +++ b/tests/regression/results.csv @@ -0,0 +1,448 @@ +Data, Config, Method, Total compressed size +silesia.tar, level -5, compress simple, 7160438 +silesia.tar, level -3, compress simple, 6789024 +silesia.tar, level -1, compress simple, 6195462 +silesia.tar, level 0, compress simple, 4875008 +silesia.tar, level 1, compress simple, 5339697 +silesia.tar, level 3, compress simple, 4875008 +silesia.tar, level 4, compress simple, 4813507 +silesia.tar, level 5, compress simple, 4722235 +silesia.tar, level 6, compress simple, 4672194 +silesia.tar, level 7, compress simple, 4606658 +silesia.tar, level 9, compress simple, 4554098 +silesia.tar, level 13, compress simple, 4491702 +silesia.tar, level 16, compress simple, 4381277 +silesia.tar, level 19, compress simple, 4281514 +silesia, level -5, compress cctx, 7152294 +silesia, level -3, compress cctx, 6789969 +silesia, level -1, compress cctx, 6191548 +silesia, level 0, compress cctx, 4862377 +silesia, level 1, compress cctx, 5318036 +silesia, level 3, compress cctx, 4862377 +silesia, level 4, compress cctx, 4800629 +silesia, level 5, compress cctx, 4710178 +silesia, level 6, compress cctx, 4659996 +silesia, level 7, compress cctx, 4596234 +silesia, level 9, compress cctx, 4543862 +silesia, level 13, compress cctx, 4482073 +silesia, level 16, compress cctx, 4377391 +silesia, level 19, compress cctx, 4293262 +silesia, long distance mode, compress cctx, 4862377 +silesia, multithreaded, compress cctx, 4862377 +silesia, multithreaded long distance mode, compress cctx, 4862377 +silesia, small window log, compress cctx, 7115734 +silesia, small hash log, compress cctx, 6554898 +silesia, small chain log, compress cctx, 4931093 +silesia, explicit params, compress cctx, 4813352 +github, level -5, compress cctx, 232744 +github, level -5 with dict, compress cctx, 47294 +github, level -3, compress cctx, 220611 +github, level -3 with dict, compress cctx, 48047 +github, level -1, compress cctx, 176575 +github, level -1 with dict, compress cctx, 43527 +github, level 0, compress cctx, 136397 +github, level 0 with dict, compress cctx, 41536 +github, level 1, compress cctx, 143457 +github, level 1 with dict, compress cctx, 42157 +github, level 3, compress cctx, 136397 +github, level 3 with dict, compress cctx, 41536 +github, level 4, compress cctx, 136144 +github, level 4 with dict, compress cctx, 41721 +github, level 5, compress cctx, 135106 +github, level 5 with dict, compress cctx, 38934 +github, level 6, compress cctx, 135108 +github, level 6 with dict, compress cctx, 38628 +github, level 7, compress cctx, 135108 +github, level 7 with dict, compress cctx, 38741 +github, level 9, compress cctx, 135108 +github, level 9 with dict, compress cctx, 39335 +github, level 13, compress cctx, 133717 +github, level 13 with dict, compress cctx, 39923 +github, level 16, compress cctx, 133717 +github, level 16 with dict, compress cctx, 37568 +github, level 19, compress cctx, 133717 +github, level 19 with dict, compress cctx, 37567 +github, long distance mode, compress cctx, decompression error +github, multithreaded, compress cctx, decompression error +github, multithreaded long distance mode, compress cctx, decompression error +github, small window log, compress cctx, decompression error +github, small hash log, compress cctx, decompression error +github, small chain log, compress cctx, decompression error +github, explicit params, compress cctx, decompression error +silesia, level -5, zstdcli, 7152342 +silesia, level -3, zstdcli, 6790021 +silesia, level -1, zstdcli, 6191597 +silesia, level 0, zstdcli, 4862425 +silesia, level 1, zstdcli, 5318084 +silesia, level 3, zstdcli, 4862425 +silesia, level 4, zstdcli, 4800677 +silesia, level 5, zstdcli, 4710226 +silesia, level 6, zstdcli, 4660044 +silesia, level 7, zstdcli, 4596282 +silesia, level 9, zstdcli, 4543910 +silesia, level 13, zstdcli, 4482121 +silesia, level 16, zstdcli, 4377439 +silesia, level 19, zstdcli, 4293310 +silesia, long distance mode, zstdcli, 4853437 +silesia, multithreaded, zstdcli, 4862425 +silesia, multithreaded long distance mode, zstdcli, 4853437 +silesia, small window log, zstdcli, 7126434 +silesia, small hash log, zstdcli, 6554946 +silesia, small chain log, zstdcli, 4931141 +silesia, explicit params, zstdcli, 4815380 +silesia.tar, level -5, zstdcli, 7161160 +silesia.tar, level -3, zstdcli, 6789865 +silesia.tar, level -1, zstdcli, 6196433 +silesia.tar, level 0, zstdcli, 4875136 +silesia.tar, level 1, zstdcli, 5340573 +silesia.tar, level 3, zstdcli, 4875136 +silesia.tar, level 4, zstdcli, 4814531 +silesia.tar, level 5, zstdcli, 4723284 +silesia.tar, level 6, zstdcli, 4673591 +silesia.tar, level 7, zstdcli, 4608342 +silesia.tar, level 9, zstdcli, 4554700 +silesia.tar, level 13, zstdcli, 4491706 +silesia.tar, level 16, zstdcli, 4381281 +silesia.tar, level 19, zstdcli, 4281518 +silesia.tar, no source size, zstdcli, 4875132 +silesia.tar, long distance mode, zstdcli, 4866975 +silesia.tar, multithreaded, zstdcli, 4875136 +silesia.tar, multithreaded long distance mode, zstdcli, 4866975 +silesia.tar, small window log, zstdcli, 7130434 +silesia.tar, small hash log, zstdcli, 6587841 +silesia.tar, small chain log, zstdcli, 4943259 +silesia.tar, explicit params, zstdcli, 4839202 +github, level -5, zstdcli, 234744 +github, level -5 with dict, zstdcli, 48718 +github, level -3, zstdcli, 222611 +github, level -3 with dict, zstdcli, 47395 +github, level -1, zstdcli, 178575 +github, level -1 with dict, zstdcli, 45170 +github, level 0, zstdcli, 138397 +github, level 0 with dict, zstdcli, 43170 +github, level 1, zstdcli, 145457 +github, level 1 with dict, zstdcli, 43682 +github, level 3, zstdcli, 138397 +github, level 3 with dict, zstdcli, 43170 +github, level 4, zstdcli, 138144 +github, level 4 with dict, zstdcli, 43306 +github, level 5, zstdcli, 137106 +github, level 5 with dict, zstdcli, 40938 +github, level 6, zstdcli, 137108 +github, level 6 with dict, zstdcli, 40632 +github, level 7, zstdcli, 137108 +github, level 7 with dict, zstdcli, 40766 +github, level 9, zstdcli, 137108 +github, level 9 with dict, zstdcli, 41326 +github, level 13, zstdcli, 135717 +github, level 13 with dict, zstdcli, 41716 +github, level 16, zstdcli, 135717 +github, level 16 with dict, zstdcli, 39577 +github, level 19, zstdcli, 135717 +github, level 19 with dict, zstdcli, 39576 +github, long distance mode, zstdcli, 138397 +github, multithreaded, zstdcli, 138397 +github, multithreaded long distance mode, zstdcli, 138397 +github, small window log, zstdcli, 138397 +github, small hash log, zstdcli, 137467 +github, small chain log, zstdcli, 138314 +github, explicit params, zstdcli, 136140 +silesia, level -5, advanced one pass, 7152294 +silesia, level -3, advanced one pass, 6789969 +silesia, level -1, advanced one pass, 6191548 +silesia, level 0, advanced one pass, 4862377 +silesia, level 1, advanced one pass, 5318036 +silesia, level 3, advanced one pass, 4862377 +silesia, level 4, advanced one pass, 4800629 +silesia, level 5, advanced one pass, 4710178 +silesia, level 6, advanced one pass, 4659996 +silesia, level 7, advanced one pass, 4596234 +silesia, level 9, advanced one pass, 4543862 +silesia, level 13, advanced one pass, 4482073 +silesia, level 16, advanced one pass, 4377391 +silesia, level 19, advanced one pass, 4293262 +silesia, no source size, advanced one pass, 4862377 +silesia, long distance mode, advanced one pass, 4853389 +silesia, multithreaded, advanced one pass, 4862377 +silesia, multithreaded long distance mode, advanced one pass, 4853389 +silesia, small window log, advanced one pass, 7126386 +silesia, small hash log, advanced one pass, 6554898 +silesia, small chain log, advanced one pass, 4931093 +silesia, explicit params, advanced one pass, 4815369 +silesia.tar, level -5, advanced one pass, 7160438 +silesia.tar, level -3, advanced one pass, 6789024 +silesia.tar, level -1, advanced one pass, 6195462 +silesia.tar, level 0, advanced one pass, 4875008 +silesia.tar, level 1, advanced one pass, 5339697 +silesia.tar, level 3, advanced one pass, 4875008 +silesia.tar, level 4, advanced one pass, 4813507 +silesia.tar, level 5, advanced one pass, 4722235 +silesia.tar, level 6, advanced one pass, 4672194 +silesia.tar, level 7, advanced one pass, 4606658 +silesia.tar, level 9, advanced one pass, 4554098 +silesia.tar, level 13, advanced one pass, 4491702 +silesia.tar, level 16, advanced one pass, 4381277 +silesia.tar, level 19, advanced one pass, 4281514 +silesia.tar, no source size, advanced one pass, 4875008 +silesia.tar, long distance mode, advanced one pass, 4861218 +silesia.tar, multithreaded, advanced one pass, 4874631 +silesia.tar, multithreaded long distance mode, advanced one pass, 4860683 +silesia.tar, small window log, advanced one pass, 7130394 +silesia.tar, small hash log, advanced one pass, 6587833 +silesia.tar, small chain log, advanced one pass, 4943255 +silesia.tar, explicit params, advanced one pass, 4829974 +github, level -5, advanced one pass, 232744 +github, level -5 with dict, advanced one pass, 46718 +github, level -3, advanced one pass, 220611 +github, level -3 with dict, advanced one pass, 45395 +github, level -1, advanced one pass, 176575 +github, level -1 with dict, advanced one pass, 43170 +github, level 0, advanced one pass, 136397 +github, level 0 with dict, advanced one pass, 41170 +github, level 1, advanced one pass, 143457 +github, level 1 with dict, advanced one pass, 41682 +github, level 3, advanced one pass, 136397 +github, level 3 with dict, advanced one pass, 41170 +github, level 4, advanced one pass, 136144 +github, level 4 with dict, advanced one pass, 41306 +github, level 5, advanced one pass, 135106 +github, level 5 with dict, advanced one pass, 38938 +github, level 6, advanced one pass, 135108 +github, level 6 with dict, advanced one pass, 38632 +github, level 7, advanced one pass, 135108 +github, level 7 with dict, advanced one pass, 38766 +github, level 9, advanced one pass, 135108 +github, level 9 with dict, advanced one pass, 39326 +github, level 13, advanced one pass, 133717 +github, level 13 with dict, advanced one pass, 39716 +github, level 16, advanced one pass, 133717 +github, level 16 with dict, advanced one pass, 37577 +github, level 19, advanced one pass, 133717 +github, level 19 with dict, advanced one pass, 37576 +github, no source size, advanced one pass, 136397 +github, long distance mode, advanced one pass, 136397 +github, multithreaded, advanced one pass, 136397 +github, multithreaded long distance mode, advanced one pass, 136397 +github, small window log, advanced one pass, 136397 +github, small hash log, advanced one pass, 135467 +github, small chain log, advanced one pass, 136314 +github, explicit params, advanced one pass, 137670 +silesia, level -5, advanced one pass small out, 7152294 +silesia, level -3, advanced one pass small out, 6789969 +silesia, level -1, advanced one pass small out, 6191548 +silesia, level 0, advanced one pass small out, 4862377 +silesia, level 1, advanced one pass small out, 5318036 +silesia, level 3, advanced one pass small out, 4862377 +silesia, level 4, advanced one pass small out, 4800629 +silesia, level 5, advanced one pass small out, 4710178 +silesia, level 6, advanced one pass small out, 4659996 +silesia, level 7, advanced one pass small out, 4596234 +silesia, level 9, advanced one pass small out, 4543862 +silesia, level 13, advanced one pass small out, 4482073 +silesia, level 16, advanced one pass small out, 4377391 +silesia, level 19, advanced one pass small out, 4293262 +silesia, no source size, advanced one pass small out, 4862377 +silesia, long distance mode, advanced one pass small out, 4853389 +silesia, multithreaded, advanced one pass small out, 4862377 +silesia, multithreaded long distance mode, advanced one pass small out, 4853389 +silesia, small window log, advanced one pass small out, 7126386 +silesia, small hash log, advanced one pass small out, 6554898 +silesia, small chain log, advanced one pass small out, 4931093 +silesia, explicit params, advanced one pass small out, 4815369 +silesia.tar, level -5, advanced one pass small out, 7160438 +silesia.tar, level -3, advanced one pass small out, 6789024 +silesia.tar, level -1, advanced one pass small out, 6195462 +silesia.tar, level 0, advanced one pass small out, 4875008 +silesia.tar, level 1, advanced one pass small out, 5339697 +silesia.tar, level 3, advanced one pass small out, 4875008 +silesia.tar, level 4, advanced one pass small out, 4813507 +silesia.tar, level 5, advanced one pass small out, 4722235 +silesia.tar, level 6, advanced one pass small out, 4672194 +silesia.tar, level 7, advanced one pass small out, 4606658 +silesia.tar, level 9, advanced one pass small out, 4554098 +silesia.tar, level 13, advanced one pass small out, 4491702 +silesia.tar, level 16, advanced one pass small out, 4381277 +silesia.tar, level 19, advanced one pass small out, 4281514 +silesia.tar, no source size, advanced one pass small out, 4875008 +silesia.tar, long distance mode, advanced one pass small out, 4861218 +silesia.tar, multithreaded, advanced one pass small out, 4874631 +silesia.tar, multithreaded long distance mode, advanced one pass small out, 4860683 +silesia.tar, small window log, advanced one pass small out, 7130394 +silesia.tar, small hash log, advanced one pass small out, 6587833 +silesia.tar, small chain log, advanced one pass small out, 4943255 +silesia.tar, explicit params, advanced one pass small out, 4829974 +github, level -5, advanced one pass small out, 232744 +github, level -5 with dict, advanced one pass small out, 46718 +github, level -3, advanced one pass small out, 220611 +github, level -3 with dict, advanced one pass small out, 45395 +github, level -1, advanced one pass small out, 176575 +github, level -1 with dict, advanced one pass small out, 43170 +github, level 0, advanced one pass small out, 136397 +github, level 0 with dict, advanced one pass small out, 41170 +github, level 1, advanced one pass small out, 143457 +github, level 1 with dict, advanced one pass small out, 41682 +github, level 3, advanced one pass small out, 136397 +github, level 3 with dict, advanced one pass small out, 41170 +github, level 4, advanced one pass small out, 136144 +github, level 4 with dict, advanced one pass small out, 41306 +github, level 5, advanced one pass small out, 135106 +github, level 5 with dict, advanced one pass small out, 38938 +github, level 6, advanced one pass small out, 135108 +github, level 6 with dict, advanced one pass small out, 38632 +github, level 7, advanced one pass small out, 135108 +github, level 7 with dict, advanced one pass small out, 38766 +github, level 9, advanced one pass small out, 135108 +github, level 9 with dict, advanced one pass small out, 39326 +github, level 13, advanced one pass small out, 133717 +github, level 13 with dict, advanced one pass small out, 39716 +github, level 16, advanced one pass small out, 133717 +github, level 16 with dict, advanced one pass small out, 37577 +github, level 19, advanced one pass small out, 133717 +github, level 19 with dict, advanced one pass small out, 37576 +github, no source size, advanced one pass small out, 136397 +github, long distance mode, advanced one pass small out, 136397 +github, multithreaded, advanced one pass small out, 136397 +github, multithreaded long distance mode, advanced one pass small out, 136397 +github, small window log, advanced one pass small out, 136397 +github, small hash log, advanced one pass small out, 135467 +github, small chain log, advanced one pass small out, 136314 +github, explicit params, advanced one pass small out, 137670 +silesia, level -5, advanced streaming, 7152294 +silesia, level -3, advanced streaming, 6789973 +silesia, level -1, advanced streaming, 6191549 +silesia, level 0, advanced streaming, 4862377 +silesia, level 1, advanced streaming, 5318036 +silesia, level 3, advanced streaming, 4862377 +silesia, level 4, advanced streaming, 4800629 +silesia, level 5, advanced streaming, 4710178 +silesia, level 6, advanced streaming, 4659996 +silesia, level 7, advanced streaming, 4596234 +silesia, level 9, advanced streaming, 4543862 +silesia, level 13, advanced streaming, 4482073 +silesia, level 16, advanced streaming, 4377391 +silesia, level 19, advanced streaming, 4293262 +silesia, no source size, advanced streaming, 4862341 +silesia, long distance mode, advanced streaming, 4853389 +silesia, multithreaded, advanced streaming, 4862377 +silesia, multithreaded long distance mode, advanced streaming, 4853389 +silesia, small window log, advanced streaming, 7126389 +silesia, small hash log, advanced streaming, 6554898 +silesia, small chain log, advanced streaming, 4931093 +silesia, explicit params, advanced streaming, 4815380 +silesia.tar, level -5, advanced streaming, 7160440 +silesia.tar, level -3, advanced streaming, 6789026 +silesia.tar, level -1, advanced streaming, 6195465 +silesia.tar, level 0, advanced streaming, 4875010 +silesia.tar, level 1, advanced streaming, 5339701 +silesia.tar, level 3, advanced streaming, 4875010 +silesia.tar, level 4, advanced streaming, 4813507 +silesia.tar, level 5, advanced streaming, 4722240 +silesia.tar, level 6, advanced streaming, 4672203 +silesia.tar, level 7, advanced streaming, 4606658 +silesia.tar, level 9, advanced streaming, 4554105 +silesia.tar, level 13, advanced streaming, 4491703 +silesia.tar, level 16, advanced streaming, 4381277 +silesia.tar, level 19, advanced streaming, 4281514 +silesia.tar, no source size, advanced streaming, 4875006 +silesia.tar, long distance mode, advanced streaming, 4861218 +silesia.tar, multithreaded, advanced streaming, 4875132 +silesia.tar, multithreaded long distance mode, advanced streaming, 4866971 +silesia.tar, small window log, advanced streaming, 7130394 +silesia.tar, small hash log, advanced streaming, 6587834 +silesia.tar, small chain log, advanced streaming, 4943260 +silesia.tar, explicit params, advanced streaming, 4830002 +github, level -5, advanced streaming, 232744 +github, level -5 with dict, advanced streaming, 46718 +github, level -3, advanced streaming, 220611 +github, level -3 with dict, advanced streaming, 45395 +github, level -1, advanced streaming, 176575 +github, level -1 with dict, advanced streaming, 43170 +github, level 0, advanced streaming, 136397 +github, level 0 with dict, advanced streaming, 41170 +github, level 1, advanced streaming, 143457 +github, level 1 with dict, advanced streaming, 41682 +github, level 3, advanced streaming, 136397 +github, level 3 with dict, advanced streaming, 41170 +github, level 4, advanced streaming, 136144 +github, level 4 with dict, advanced streaming, 41306 +github, level 5, advanced streaming, 135106 +github, level 5 with dict, advanced streaming, 38938 +github, level 6, advanced streaming, 135108 +github, level 6 with dict, advanced streaming, 38632 +github, level 7, advanced streaming, 135108 +github, level 7 with dict, advanced streaming, 38766 +github, level 9, advanced streaming, 135108 +github, level 9 with dict, advanced streaming, 39326 +github, level 13, advanced streaming, 133717 +github, level 13 with dict, advanced streaming, 39716 +github, level 16, advanced streaming, 133717 +github, level 16 with dict, advanced streaming, 37577 +github, level 19, advanced streaming, 133717 +github, level 19 with dict, advanced streaming, 37576 +github, no source size, advanced streaming, 136397 +github, long distance mode, advanced streaming, 136397 +github, multithreaded, advanced streaming, 136397 +github, multithreaded long distance mode, advanced streaming, 136397 +github, small window log, advanced streaming, 136397 +github, small hash log, advanced streaming, 135467 +github, small chain log, advanced streaming, 136314 +github, explicit params, advanced streaming, 137670 +silesia, level -5, old streaming, 7152294 +silesia, level -3, old streaming, 6789973 +silesia, level -1, old streaming, 6191549 +silesia, level 0, old streaming, 4862377 +silesia, level 1, old streaming, 5318036 +silesia, level 3, old streaming, 4862377 +silesia, level 4, old streaming, 4800629 +silesia, level 5, old streaming, 4710178 +silesia, level 6, old streaming, 4659996 +silesia, level 7, old streaming, 4596234 +silesia, level 9, old streaming, 4543862 +silesia, level 13, old streaming, 4482073 +silesia, level 16, old streaming, 4377391 +silesia, level 19, old streaming, 4293262 +silesia, no source size, old streaming, 4862341 +silesia.tar, level -5, old streaming, 7160440 +silesia.tar, level -3, old streaming, 6789026 +silesia.tar, level -1, old streaming, 6195465 +silesia.tar, level 0, old streaming, 4875010 +silesia.tar, level 1, old streaming, 5339701 +silesia.tar, level 3, old streaming, 4875010 +silesia.tar, level 4, old streaming, 4813507 +silesia.tar, level 5, old streaming, 4722240 +silesia.tar, level 6, old streaming, 4672203 +silesia.tar, level 7, old streaming, 4606658 +silesia.tar, level 9, old streaming, 4554105 +silesia.tar, level 13, old streaming, 4491703 +silesia.tar, level 16, old streaming, 4381277 +silesia.tar, level 19, old streaming, 4281514 +silesia.tar, no source size, old streaming, 4875006 +github, level -5, old streaming, 232744 +github, level -5 with dict, old streaming, 46718 +github, level -3, old streaming, 220611 +github, level -3 with dict, old streaming, 45395 +github, level -1, old streaming, 176575 +github, level -1 with dict, old streaming, 43170 +github, level 0, old streaming, 136397 +github, level 0 with dict, old streaming, 41170 +github, level 1, old streaming, 143457 +github, level 1 with dict, old streaming, 41682 +github, level 3, old streaming, 136397 +github, level 3 with dict, old streaming, 41170 +github, level 4, old streaming, 136144 +github, level 4 with dict, old streaming, 41306 +github, level 5, old streaming, 135106 +github, level 5 with dict, old streaming, 38938 +github, level 6, old streaming, 135108 +github, level 6 with dict, old streaming, 38632 +github, level 7, old streaming, 135108 +github, level 7 with dict, old streaming, 38766 +github, level 9, old streaming, 135108 +github, level 9 with dict, old streaming, 39326 +github, level 13, old streaming, 133717 +github, level 13 with dict, old streaming, 39716 +github, level 16, old streaming, 133717 +github, level 16 with dict, old streaming, 37577 +github, level 19, old streaming, 133717 +github, level 19 with dict, old streaming, 37576 +github, no source size, old streaming, 141003 diff --git a/tests/regression/test.c b/tests/regression/test.c new file mode 100644 index 000000000000..9e7b83c19daf --- /dev/null +++ b/tests/regression/test.c @@ -0,0 +1,362 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include <assert.h> +#include <getopt.h> +#include <stdio.h> +#include <string.h> + +#include "config.h" +#include "data.h" +#include "method.h" + +static int g_max_name_len = 0; + +/** Check if a name contains a comma or is too long. */ +static int is_name_bad(char const* name) { + if (name == NULL) + return 1; + int const len = strlen(name); + if (len > g_max_name_len) + g_max_name_len = len; + for (; *name != '\0'; ++name) + if (*name == ',') + return 1; + return 0; +} + +/** Check if any of the names contain a comma. */ +static int are_names_bad() { + for (size_t method = 0; methods[method] != NULL; ++method) + if (is_name_bad(methods[method]->name)) { + fprintf(stderr, "method name %s is bad\n", methods[method]->name); + return 1; + } + for (size_t datum = 0; data[datum] != NULL; ++datum) + if (is_name_bad(data[datum]->name)) { + fprintf(stderr, "data name %s is bad\n", data[datum]->name); + return 1; + } + for (size_t config = 0; configs[config] != NULL; ++config) + if (is_name_bad(configs[config]->name)) { + fprintf(stderr, "config name %s is bad\n", configs[config]->name); + return 1; + } + return 0; +} + +/** + * Option parsing using getopt. + * When you add a new option update: long_options, long_extras, and + * short_options. + */ + +/** Option variables filled by parse_args. */ +static char const* g_output = NULL; +static char const* g_diff = NULL; +static char const* g_cache = NULL; +static char const* g_zstdcli = NULL; +static char const* g_config = NULL; +static char const* g_data = NULL; +static char const* g_method = NULL; + +typedef enum { + required_option, + optional_option, + help_option, +} option_type; + +/** + * Extra state that we need to keep per-option that we can't store in getopt. + */ +struct option_extra { + int id; /**< The short option name, used as an id. */ + char const* help; /**< The help message. */ + option_type opt_type; /**< The option type: required, optional, or help. */ + char const** value; /**< The value to set or NULL if no_argument. */ +}; + +/** The options. */ +static struct option long_options[] = { + {"cache", required_argument, NULL, 'c'}, + {"output", required_argument, NULL, 'o'}, + {"zstd", required_argument, NULL, 'z'}, + {"config", required_argument, NULL, 128}, + {"data", required_argument, NULL, 129}, + {"method", required_argument, NULL, 130}, + {"diff", required_argument, NULL, 'd'}, + {"help", no_argument, NULL, 'h'}, +}; + +static size_t const nargs = sizeof(long_options) / sizeof(long_options[0]); + +/** The extra info for the options. Must be in the same order as the options. */ +static struct option_extra long_extras[] = { + {'c', "the cache directory", required_option, &g_cache}, + {'o', "write the results here", required_option, &g_output}, + {'z', "zstd cli tool", required_option, &g_zstdcli}, + {128, "use this config", optional_option, &g_config}, + {129, "use this data", optional_option, &g_data}, + {130, "use this method", optional_option, &g_method}, + {'d', "compare the results to this file", optional_option, &g_diff}, + {'h', "display this message", help_option, NULL}, +}; + +/** The short options. Must correspond to the options. */ +static char const short_options[] = "c:d:ho:z:"; + +/** Return the help string for the option type. */ +static char const* required_message(option_type opt_type) { + switch (opt_type) { + case required_option: + return "[required]"; + case optional_option: + return "[optional]"; + case help_option: + return ""; + default: + assert(0); + return NULL; + } +} + +/** Print the help for the program. */ +static void print_help(void) { + fprintf(stderr, "regression test runner\n"); + size_t const nargs = sizeof(long_options) / sizeof(long_options[0]); + for (size_t i = 0; i < nargs; ++i) { + if (long_options[i].val < 128) { + /* Long / short - help [option type] */ + fprintf( + stderr, + "--%s / -%c \t- %s %s\n", + long_options[i].name, + long_options[i].val, + long_extras[i].help, + required_message(long_extras[i].opt_type)); + } else { + /* Short / long - help [option type] */ + fprintf( + stderr, + "--%s \t- %s %s\n", + long_options[i].name, + long_extras[i].help, + required_message(long_extras[i].opt_type)); + } + } +} + +/** Parse the arguments. Teturn 0 on success. Print help on failure. */ +static int parse_args(int argc, char** argv) { + int option_index = 0; + int c; + + while (1) { + c = getopt_long(argc, argv, short_options, long_options, &option_index); + if (c == -1) + break; + + int found = 0; + for (size_t i = 0; i < nargs; ++i) { + if (c == long_extras[i].id && long_extras[i].value != NULL) { + *long_extras[i].value = optarg; + found = 1; + break; + } + } + if (found) + continue; + + switch (c) { + case 'h': + case '?': + default: + print_help(); + return 1; + } + } + + int bad = 0; + for (size_t i = 0; i < nargs; ++i) { + if (long_extras[i].opt_type != required_option) + continue; + if (long_extras[i].value == NULL) + continue; + if (*long_extras[i].value != NULL) + continue; + fprintf( + stderr, + "--%s is a required argument but is not set\n", + long_options[i].name); + bad = 1; + } + if (bad) { + fprintf(stderr, "\n"); + print_help(); + return 1; + } + + return 0; +} + +/** Helper macro to print to stderr and a file. */ +#define tprintf(file, ...) \ + do { \ + fprintf(file, __VA_ARGS__); \ + fprintf(stderr, __VA_ARGS__); \ + } while (0) +/** Helper macro to flush stderr and a file. */ +#define tflush(file) \ + do { \ + fflush(file); \ + fflush(stderr); \ + } while (0) + +void tprint_names( + FILE* results, + char const* data_name, + char const* config_name, + char const* method_name) { + int const data_padding = g_max_name_len - strlen(data_name); + int const config_padding = g_max_name_len - strlen(config_name); + int const method_padding = g_max_name_len - strlen(method_name); + + tprintf( + results, + "%s, %*s%s, %*s%s, %*s", + data_name, + data_padding, + "", + config_name, + config_padding, + "", + method_name, + method_padding, + ""); +} + +/** + * Run all the regression tests and record the results table to results and + * stderr progressively. + */ +static int run_all(FILE* results) { + tprint_names(results, "Data", "Config", "Method"); + tprintf(results, "Total compressed size\n"); + for (size_t method = 0; methods[method] != NULL; ++method) { + if (g_method != NULL && strcmp(methods[method]->name, g_method)) + continue; + for (size_t datum = 0; data[datum] != NULL; ++datum) { + if (g_data != NULL && strcmp(data[datum]->name, g_data)) + continue; + /* Create the state common to all configs */ + method_state_t* state = methods[method]->create(data[datum]); + for (size_t config = 0; configs[config] != NULL; ++config) { + if (g_config != NULL && strcmp(configs[config]->name, g_config)) + continue; + if (config_skip_data(configs[config], data[datum])) + continue; + /* Print the result for the (method, data, config) tuple. */ + result_t const result = + methods[method]->compress(state, configs[config]); + if (result_is_skip(result)) + continue; + tprint_names( + results, + data[datum]->name, + configs[config]->name, + methods[method]->name); + if (result_is_error(result)) { + tprintf(results, "%s\n", result_get_error_string(result)); + } else { + tprintf( + results, + "%llu\n", + (unsigned long long)result_get_data(result).total_size); + } + tflush(results); + } + methods[method]->destroy(state); + } + } + return 0; +} + +/** memcmp() the old results file and the new results file. */ +static int diff_results(char const* actual_file, char const* expected_file) { + data_buffer_t const actual = data_buffer_read(actual_file); + data_buffer_t const expected = data_buffer_read(expected_file); + int ret = 1; + + if (actual.data == NULL) { + fprintf(stderr, "failed to open results '%s' for diff\n", actual_file); + goto out; + } + if (expected.data == NULL) { + fprintf( + stderr, + "failed to open previous results '%s' for diff\n", + expected_file); + goto out; + } + + ret = data_buffer_compare(actual, expected); + if (ret != 0) { + fprintf( + stderr, + "actual results '%s' does not match expected results '%s'\n", + actual_file, + expected_file); + } else { + fprintf(stderr, "actual results match expected results\n"); + } +out: + data_buffer_free(actual); + data_buffer_free(expected); + return ret; +} + +int main(int argc, char** argv) { + /* Parse args and validate modules. */ + int ret = parse_args(argc, argv); + if (ret != 0) + return ret; + + if (are_names_bad()) + return 1; + + /* Initialize modules. */ + method_set_zstdcli(g_zstdcli); + ret = data_init(g_cache); + if (ret != 0) { + fprintf(stderr, "data_init() failed with error=%s\n", strerror(ret)); + return 1; + } + + /* Run the regression tests. */ + ret = 1; + FILE* results = fopen(g_output, "w"); + if (results == NULL) { + fprintf(stderr, "Failed to open the output file\n"); + goto out; + } + ret = run_all(results); + fclose(results); + + if (ret != 0) + goto out; + + if (g_diff) + /* Diff the new results with the previous results. */ + ret = diff_results(g_output, g_diff); + +out: + data_finish(); + return ret; +} diff --git a/tests/roundTripCrash.c b/tests/roundTripCrash.c index 90afcd4b2a8b..3f4ace8c9800 100644 --- a/tests/roundTripCrash.c +++ b/tests/roundTripCrash.c @@ -85,7 +85,7 @@ static size_t cctxParamRoundTripTest(void* resultBuff, size_t resultBuffCapacity ZSTD_CCtx* const cctx = ZSTD_createCCtx(); ZSTD_CCtx_params* const cctxParams = ZSTD_createCCtxParams(); ZSTD_inBuffer inBuffer = { srcBuff, srcBuffSize, 0 }; - ZSTD_outBuffer outBuffer = {compressedBuff, compressedBuffCapacity, 0 }; + ZSTD_outBuffer outBuffer = { compressedBuff, compressedBuffCapacity, 0 }; static const int maxClevel = 19; size_t const hashLength = MIN(128, srcBuffSize); @@ -93,15 +93,15 @@ static size_t cctxParamRoundTripTest(void* resultBuff, size_t resultBuffCapacity int const cLevel = h32 % maxClevel; /* Set parameters */ - CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_compressionLevel, cLevel) ); - CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_nbWorkers, 2) ); - CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_overlapSizeLog, 5) ); + CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_c_compressionLevel, cLevel) ); + CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_c_nbWorkers, 2) ); + CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_c_overlapLog, 5) ); /* Apply parameters */ CHECK_Z( ZSTD_CCtx_setParametersUsingCCtxParams(cctx, cctxParams) ); - CHECK_Z (ZSTD_compress_generic(cctx, &outBuffer, &inBuffer, ZSTD_e_end) ); + CHECK_Z (ZSTD_compressStream2(cctx, &outBuffer, &inBuffer, ZSTD_e_end) ); ZSTD_freeCCtxParams(cctxParams); ZSTD_freeCCtx(cctx); diff --git a/tests/seqgen.c b/tests/seqgen.c index 8233ecee1cfd..06e3ed5ba0b0 100644 --- a/tests/seqgen.c +++ b/tests/seqgen.c @@ -17,7 +17,7 @@ static const size_t kMatchBytes = 128; #define SEQ_rotl32(x,r) ((x << r) | (x >> (32 - r))) -static BYTE SEQ_randByte(U32* src) +static BYTE SEQ_randByte(unsigned* src) { static const U32 prime1 = 2654435761U; static const U32 prime2 = 2246822519U; diff --git a/tests/symbols.c b/tests/symbols.c index b37082131464..600d81670a55 100644 --- a/tests/symbols.c +++ b/tests/symbols.c @@ -89,7 +89,6 @@ static const void *symbols[] = { &ZSTD_sizeof_CStream, &ZSTD_createDStream_advanced, &ZSTD_initDStream_usingDict, - &ZSTD_setDStreamParameter, &ZSTD_initDStream_usingDDict, &ZSTD_resetDStream, &ZSTD_sizeof_DStream, diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index f47451a3c3d5..ac300890415f 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -46,7 +46,7 @@ #define MB *(1U<<20) #define GB *(1U<<30) -static const U32 nbTestsDefault = 10000; +static const int nbTestsDefault = 10000; static const U32 g_cLevelMax_smallTests = 10; #define COMPRESSIBLE_NOISE_LENGTH (10 MB) #define FUZ_COMPRESSIBILITY_DEFAULT 50 @@ -84,7 +84,7 @@ static U64 g_clockTime = 0; @return : a 27 bits random value, from a 32-bits `seed`. `seed` is also modified */ #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) -static unsigned int FUZ_rand(unsigned int* seedPtr) +static U32 FUZ_rand(U32* seedPtr) { static const U32 prime2 = 2246822519U; U32 rand32 = *seedPtr; @@ -100,7 +100,7 @@ static unsigned int FUZ_rand(unsigned int* seedPtr) DISPLAY("Error => "); \ DISPLAY(__VA_ARGS__); \ DISPLAY(" (seed %u, test nb %u, line %u) \n", \ - seed, testNb, __LINE__); \ + (unsigned)seed, testNb, __LINE__); \ goto _output_error; \ } } @@ -175,11 +175,11 @@ static size_t SEQ_roundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, size_t cret; do { - ZSTD_outBuffer cout = {compressed, sizeof(compressed), 0}; - ZSTD_inBuffer din = {compressed, 0, 0}; - ZSTD_outBuffer dout = {uncompressed, 0, 0}; + ZSTD_outBuffer cout = { compressed, sizeof(compressed), 0 }; + ZSTD_inBuffer din = { compressed, 0, 0 }; + ZSTD_outBuffer dout = { uncompressed, 0, 0 }; - cret = ZSTD_compress_generic(cctx, &cout, &cin, endOp); + cret = ZSTD_compressStream2(cctx, &cout, &cin, endOp); if (ZSTD_isError(cret)) return cret; @@ -223,19 +223,19 @@ static size_t SEQ_generateRoundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, static size_t getCCtxParams(ZSTD_CCtx* zc, ZSTD_parameters* savedParams) { - unsigned value; - CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_windowLog, &savedParams->cParams.windowLog)); - CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_hashLog, &savedParams->cParams.hashLog)); - CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_chainLog, &savedParams->cParams.chainLog)); - CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_searchLog, &savedParams->cParams.searchLog)); - CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_minMatch, &savedParams->cParams.searchLength)); - CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_targetLength, &savedParams->cParams.targetLength)); - CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_compressionStrategy, &value)); + int value; + CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_windowLog, (int*)&savedParams->cParams.windowLog)); + CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_hashLog, (int*)&savedParams->cParams.hashLog)); + CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_chainLog, (int*)&savedParams->cParams.chainLog)); + CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_searchLog, (int*)&savedParams->cParams.searchLog)); + CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_minMatch, (int*)&savedParams->cParams.minMatch)); + CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_targetLength, (int*)&savedParams->cParams.targetLength)); + CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_strategy, &value)); savedParams->cParams.strategy = value; - CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_checksumFlag, &savedParams->fParams.checksumFlag)); - CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_contentSizeFlag, &savedParams->fParams.contentSizeFlag)); - CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_dictIDFlag, &value)); + CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_checksumFlag, &savedParams->fParams.checksumFlag)); + CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_contentSizeFlag, &savedParams->fParams.contentSizeFlag)); + CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_dictIDFlag, &value)); savedParams->fParams.noDictIDFlag = !value; return 0; } @@ -248,7 +248,7 @@ static U32 badParameters(ZSTD_CCtx* zc, ZSTD_parameters const savedParams) CHECK_RET(2, params.cParams.hashLog != savedParams.cParams.hashLog, "hashLog"); CHECK_RET(3, params.cParams.chainLog != savedParams.cParams.chainLog, "chainLog"); CHECK_RET(4, params.cParams.searchLog != savedParams.cParams.searchLog, "searchLog"); - CHECK_RET(5, params.cParams.searchLength != savedParams.cParams.searchLength, "searchLength"); + CHECK_RET(5, params.cParams.minMatch != savedParams.cParams.minMatch, "minMatch"); CHECK_RET(6, params.cParams.targetLength != savedParams.cParams.targetLength, "targetLength"); CHECK_RET(7, params.fParams.checksumFlag != savedParams.fParams.checksumFlag, "checksumFlag"); @@ -268,7 +268,7 @@ static int basicUnitTests(U32 seed, double compressibility) void* decodedBuffer = malloc(decodedBufferSize); size_t cSize; int testResult = 0; - U32 testNb = 1; + int testNb = 1; U32 coreSeed = 0; /* this name to conform with CHECK_Z macro display */ ZSTD_CStream* zc = ZSTD_createCStream(); ZSTD_DStream* zd = ZSTD_createDStream(); @@ -309,7 +309,7 @@ static int basicUnitTests(U32 seed, double compressibility) if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ { size_t const r = ZSTD_endStream(zc, &outBuff); if (r != 0) goto _output_error; } /* error, or some data not flushed */ - DISPLAYLEVEL(3, "OK (%u bytes)\n", (U32)outBuff.pos); + DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)outBuff.pos); /* generate skippable frame */ MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START); @@ -331,7 +331,8 @@ static int basicUnitTests(U32 seed, double compressibility) { size_t const r = ZSTD_endStream(zc, &outBuff); if (r != 0) goto _output_error; } /* error, or some data not flushed */ cSize += outBuff.pos; - DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", + (unsigned)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); /* context size functions */ DISPLAYLEVEL(3, "test%3i : estimate CStream size : ", testNb++); @@ -340,20 +341,20 @@ static int basicUnitTests(U32 seed, double compressibility) size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); /* uses ZSTD_initCStream_usingDict() */ if (ZSTD_isError(cstreamSize)) goto _output_error; if (ZSTD_isError(cdictSize)) goto _output_error; - DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)(cstreamSize + cdictSize)); + DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)(cstreamSize + cdictSize)); } DISPLAYLEVEL(3, "test%3i : check actual CStream size : ", testNb++); { size_t const s = ZSTD_sizeof_CStream(zc); if (ZSTD_isError(s)) goto _output_error; - DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s); + DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s); } /* Attempt bad compression parameters */ DISPLAYLEVEL(3, "test%3i : use bad compression parameters : ", testNb++); { size_t r; ZSTD_parameters params = ZSTD_getParams(1, 0, 0); - params.cParams.searchLength = 2; + params.cParams.minMatch = 2; r = ZSTD_initCStream_advanced(zc, NULL, 0, params, 0); if (!ZSTD_isError(r)) goto _output_error; DISPLAYLEVEL(3, "init error : %s \n", ZSTD_getErrorName(r)); @@ -369,7 +370,7 @@ static int basicUnitTests(U32 seed, double compressibility) outBuff.size = CNBufferSize; outBuff.pos = 0; { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff); - DISPLAYLEVEL(5, " ( ZSTD_decompressStream => %u ) ", (U32)r); + DISPLAYLEVEL(5, " ( ZSTD_decompressStream => %u ) ", (unsigned)r); if (r != 0) goto _output_error; } if (outBuff.pos != 0) goto _output_error; /* skippable frame output len is 0 */ @@ -379,7 +380,7 @@ static int basicUnitTests(U32 seed, double compressibility) inBuff2 = inBuff; DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize); - CHECK_Z( ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1000000000) ); /* large limit */ + CHECK_Z( ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, ZSTD_WINDOWLOG_LIMIT_DEFAULT+1) ); /* large limit */ { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff); if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */ @@ -409,18 +410,18 @@ static int basicUnitTests(U32 seed, double compressibility) const void* cStart = (char*)compressedBuffer + (skippableFrameSize + 8); size_t const gfhError = ZSTD_getFrameHeader(&fhi, cStart, cSize); if (gfhError!=0) goto _output_error; - DISPLAYLEVEL(5, " (windowSize : %u) ", (U32)fhi.windowSize); + DISPLAYLEVEL(5, " (windowSize : %u) ", (unsigned)fhi.windowSize); { size_t const s = ZSTD_estimateDStreamSize(fhi.windowSize) /* uses ZSTD_initDStream_usingDict() */ + ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy); if (ZSTD_isError(s)) goto _output_error; - DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s); + DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s); } } DISPLAYLEVEL(3, "test%3i : check actual DStream size : ", testNb++); { size_t const s = ZSTD_sizeof_DStream(zd); if (ZSTD_isError(s)) goto _output_error; - DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s); + DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s); } /* Decompression by small increment */ @@ -507,7 +508,7 @@ static int basicUnitTests(U32 seed, double compressibility) if (r != 0) goto _output_error; } /* error, or some data not flushed */ { unsigned long long origSize = ZSTD_findDecompressedSize(outBuff.dst, outBuff.pos); if ((size_t)origSize != CNBufferSize) goto _output_error; } /* exact original size must be present */ - DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); /* wrong _srcSize compression test */ DISPLAYLEVEL(3, "test%3i : too large srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1); @@ -598,7 +599,7 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "test%3i : digested dictionary : ", testNb++); { ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, 1 /*byRef*/ ); size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict); - DISPLAYLEVEL(5, "ZSTD_initCStream_usingCDict result : %u ", (U32)initError); + DISPLAYLEVEL(5, "ZSTD_initCStream_usingCDict result : %u ", (unsigned)initError); if (ZSTD_isError(initError)) goto _output_error; outBuff.dst = compressedBuffer; outBuff.size = compressedBufferSize; @@ -610,18 +611,18 @@ static int basicUnitTests(U32 seed, double compressibility) CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ { size_t const r = ZSTD_endStream(zc, &outBuff); - DISPLAYLEVEL(5, "- ZSTD_endStream result : %u ", (U32)r); + DISPLAYLEVEL(5, "- ZSTD_endStream result : %u ", (unsigned)r); if (r != 0) goto _output_error; /* error, or some data not flushed */ } cSize = outBuff.pos; ZSTD_freeCDict(cdict); - DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100); } DISPLAYLEVEL(3, "test%3i : check CStream size : ", testNb++); { size_t const s = ZSTD_sizeof_CStream(zc); if (ZSTD_isError(s)) goto _output_error; - DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s); + DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s); } DISPLAYLEVEL(4, "test%3i : check Dictionary ID : ", testNb++); @@ -631,7 +632,7 @@ static int basicUnitTests(U32 seed, double compressibility) } /* DDict scenario */ - DISPLAYLEVEL(3, "test%3i : decompress %u bytes with digested dictionary : ", testNb++, (U32)CNBufferSize); + DISPLAYLEVEL(3, "test%3i : decompress %u bytes with digested dictionary : ", testNb++, (unsigned)CNBufferSize); { ZSTD_DDict* const ddict = ZSTD_createDDict(dictionary.start, dictionary.filled); size_t const initError = ZSTD_initDStream_usingDDict(zd, ddict); if (ZSTD_isError(initError)) goto _output_error; @@ -649,16 +650,10 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "OK \n"); } - /* test ZSTD_setDStreamParameter() resilience */ - DISPLAYLEVEL(3, "test%3i : wrong parameter for ZSTD_setDStreamParameter(): ", testNb++); - { size_t const r = ZSTD_setDStreamParameter(zd, (ZSTD_DStreamParameter_e)999, 1); /* large limit */ - if (!ZSTD_isError(r)) goto _output_error; } - DISPLAYLEVEL(3, "OK \n"); - /* Memory restriction */ DISPLAYLEVEL(3, "test%3i : maxWindowSize < frame requirement : ", testNb++); ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize); - CHECK_Z( ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1000) ); /* too small limit */ + CHECK_Z( ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, 10) ); /* too small limit */ outBuff.dst = decodedBuffer; outBuff.size = CNBufferSize; outBuff.pos = 0; @@ -668,7 +663,7 @@ static int basicUnitTests(U32 seed, double compressibility) { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff); if (!ZSTD_isError(r)) goto _output_error; /* must fail : frame requires > 100 bytes */ DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); } - ZSTD_DCtx_reset(zd); /* leave zd in good shape for next tests */ + ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters); /* leave zd in good shape for next tests */ DISPLAYLEVEL(3, "test%3i : dictionary source size and level : ", testNb++); { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); @@ -692,12 +687,12 @@ static int basicUnitTests(U32 seed, double compressibility) inBuff.size = size; inBuff.pos = 0; CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); - CHECK_Z(ZSTD_compress_generic(cctx, &outBuff, &inBuff, ZSTD_e_end)); + CHECK_Z(ZSTD_compressStream2(cctx, &outBuff, &inBuff, ZSTD_e_end)); CHECK(badParameters(cctx, savedParams), "Bad CCtx params"); if (inBuff.pos != inBuff.size) goto _output_error; { ZSTD_outBuffer decOut = {decodedBuffer, size, 0}; ZSTD_inBuffer decIn = {outBuff.dst, outBuff.pos, 0}; - CHECK_Z( ZSTD_decompress_generic(dctx, &decOut, &decIn) ); + CHECK_Z( ZSTD_decompressStream(dctx, &decOut, &decIn) ); if (decIn.pos != decIn.size) goto _output_error; if (decOut.pos != size) goto _output_error; { U64 const crcDec = XXH64(decOut.dst, decOut.pos, 0); @@ -729,7 +724,7 @@ static int basicUnitTests(U32 seed, double compressibility) if (r != 0) goto _output_error; } /* error, or some data not flushed */ cSize = outBuff.pos; ZSTD_freeCDict(cdict); - DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100); } DISPLAYLEVEL(3, "test%3i : try retrieving dictID from frame : ", testNb++); @@ -752,10 +747,10 @@ static int basicUnitTests(U32 seed, double compressibility) inBuff.src = CNBuffer; inBuff.size = CNBufferSize; inBuff.pos = 0; - CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end) ); + CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) ); if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ cSize = outBuff.pos; - DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100); DISPLAYLEVEL(3, "test%3i : decompress with ZSTD_DCtx_refPrefix : ", testNb++); CHECK_Z( ZSTD_DCtx_refPrefix(zd, dictionary.start, dictionary.filled) ); @@ -765,7 +760,7 @@ static int basicUnitTests(U32 seed, double compressibility) inBuff.src = compressedBuffer; inBuff.size = cSize; inBuff.pos = 0; - CHECK_Z( ZSTD_decompress_generic(zd, &outBuff, &inBuff) ); + CHECK_Z( ZSTD_decompressStream(zd, &outBuff, &inBuff) ); if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ if (outBuff.pos != CNBufferSize) goto _output_error; /* must regenerate whole input */ DISPLAYLEVEL(3, "OK \n"); @@ -776,17 +771,17 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); } - DISPLAYLEVEL(3, "test%3i : compress again with ZSTD_compress_generic : ", testNb++); + DISPLAYLEVEL(3, "test%3i : compress again with ZSTD_compressStream2 : ", testNb++); outBuff.dst = compressedBuffer; outBuff.size = compressedBufferSize; outBuff.pos = 0; inBuff.src = CNBuffer; inBuff.size = CNBufferSize; inBuff.pos = 0; - CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end) ); + CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) ); if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ cSize = outBuff.pos; - DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100); DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should work): ", testNb++); CHECK_Z( ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize) ); @@ -842,7 +837,7 @@ static int basicUnitTests(U32 seed, double compressibility) /* Basic multithreading compression test */ DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); { ZSTD_parameters const params = ZSTD_getParams(1, 0, 0); - unsigned jobSize; + int jobSize; CHECK_Z( ZSTDMT_getMTCtxParameter(mtctx, ZSTDMT_p_jobSize, &jobSize)); CHECK(jobSize != 0, "job size non-zero"); CHECK_Z( ZSTDMT_initCStream_advanced(mtctx, CNBuffer, dictSize, params, CNBufferSize) ); @@ -873,10 +868,10 @@ static int basicUnitTests(U32 seed, double compressibility) size_t const start = jobSize + (offset-1); const BYTE* const srcToCopy = (const BYTE*)CNBuffer + start; BYTE* const dst = (BYTE*)CNBuffer + start - offset; - DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads + dictionary : ", testNb++, (U32)srcSize); - CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_compressionLevel, 3) ); - CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_nbWorkers, nbWorkers) ); - CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_jobSize, jobSize) ); + DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads + dictionary : ", testNb++, (unsigned)srcSize); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 3) ); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, nbWorkers) ); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_jobSize, jobSize) ); assert(start > offset); assert(start + segLength < COMPRESSIBLE_NOISE_LENGTH); memcpy(dst, srcToCopy, segLength); /* create a long repetition at long distance for job 2 */ @@ -891,7 +886,7 @@ static int basicUnitTests(U32 seed, double compressibility) ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem); DISPLAYLEVEL(5, "cParams.windowLog = %u : ", cParams.windowLog); CHECK_Z( ZSTD_CCtx_refCDict(zc, cdict) ); - CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end) ); + CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) ); CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) ); /* do not keep a reference to cdict, as its lifetime ends */ ZSTD_freeCDict(cdict); } @@ -903,7 +898,7 @@ static int basicUnitTests(U32 seed, double compressibility) { ZSTD_DStream* const dstream = ZSTD_createDCtx(); ZSTD_frameHeader zfh; ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize); - DISPLAYLEVEL(5, "frame windowsize = %u : ", (U32)zfh.windowSize); + DISPLAYLEVEL(5, "frame windowsize = %u : ", (unsigned)zfh.windowSize); outBuff.dst = decodedBuffer; outBuff.size = CNBufferSize; outBuff.pos = 0; @@ -936,11 +931,11 @@ static int basicUnitTests(U32 seed, double compressibility) if (!cdict || !ddict) goto _output_error; - ZSTD_CCtx_reset(zc); + ZSTD_CCtx_reset(zc, ZSTD_reset_session_only); ZSTD_resetDStream(zd); CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict)); CHECK_Z(ZSTD_initDStream_usingDDict(zd, ddict)); - CHECK_Z(ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1U << kMaxWindowLog)); + CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, kMaxWindowLog)); /* Test all values < 300 */ for (value = 0; value < 300; ++value) { for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) { @@ -968,12 +963,12 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_srcSize sets requestedParams : ", testNb++); - { unsigned level; + { int level; CHECK_Z(ZSTD_initCStream_srcSize(zc, 11, ZSTD_CONTENTSIZE_UNKNOWN)); - CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_compressionLevel, &level)); + CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level)); CHECK(level != 11, "Compression level does not match"); ZSTD_resetCStream(zc, ZSTD_CONTENTSIZE_UNKNOWN); - CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_p_compressionLevel, &level)); + CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level)); CHECK(level != 11, "Compression level does not match"); } DISPLAYLEVEL(3, "OK \n"); @@ -1083,17 +1078,16 @@ static int basicUnitTests(U32 seed, double compressibility) int remainingInput = 256 * 1024; int offset; - ZSTD_CCtx_reset(zc); - CHECK_Z(ZSTD_CCtx_resetParameters(zc)); + CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters)); CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict)); - CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_p_checksumFlag, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1)); /* Write a bunch of 6 byte blocks */ while (remainingInput > 0) { char testBuffer[6] = "\xAA\xAA\xAA\xAA\xAA\xAA"; const size_t kSmallBlockSize = sizeof(testBuffer); ZSTD_inBuffer in = {testBuffer, kSmallBlockSize, 0}; - CHECK_Z(ZSTD_compress_generic(zc, &out, &in, ZSTD_e_flush)); + CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_flush)); CHECK(in.pos != in.size, "input not fully consumed"); remainingInput -= kSmallBlockSize; } @@ -1101,7 +1095,7 @@ static int basicUnitTests(U32 seed, double compressibility) for (offset = 1024; offset >= 0; offset -= 128) { ZSTD_inBuffer in = {dictionary.start + offset, 128, 0}; ZSTD_EndDirective flush = offset > 0 ? ZSTD_e_continue : ZSTD_e_end; - CHECK_Z(ZSTD_compress_generic(zc, &out, &in, flush)); + CHECK_Z(ZSTD_compressStream2(zc, &out, &in, flush)); CHECK(in.pos != in.size, "input not fully consumed"); } /* Ensure decompression works */ @@ -1139,10 +1133,10 @@ static size_t findDiff(const void* buf1, const void* buf2, size_t max) if (b1[u] != b2[u]) break; } if (u==max) { - DISPLAY("=> No difference detected within %u bytes \n", (U32)max); + DISPLAY("=> No difference detected within %u bytes \n", (unsigned)max); return u; } - DISPLAY("Error at position %u / %u \n", (U32)u, (U32)max); + DISPLAY("Error at position %u / %u \n", (unsigned)u, (unsigned)max); if (u>=3) DISPLAY(" %02X %02X %02X ", b1[u-3], b1[u-2], b1[u-1]); @@ -1175,7 +1169,7 @@ static U32 FUZ_randomClampedLength(U32* seed, U32 minVal, U32 maxVal) return (U32)((FUZ_rand(seed) % mod) + minVal); } -static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility, int bigTests) +static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, int bigTests) { U32 const maxSrcLog = bigTests ? 24 : 22; static const U32 maxSampleLog = 19; @@ -1188,7 +1182,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres size_t const dstBufferSize = srcBufferSize; BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize); U32 result = 0; - U32 testNb = 0; + unsigned testNb = 0; U32 coreSeed = seed; ZSTD_CStream* zc = ZSTD_createCStream(); /* will be re-created sometimes */ ZSTD_DStream* zd = ZSTD_createDStream(); /* will be re-created sometimes */ @@ -1369,7 +1363,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres } CHECK (decompressionResult != 0, "frame not fully decoded"); CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", - (U32)outBuff.pos, (U32)totalTestSize); + (unsigned)outBuff.pos, (unsigned)totalTestSize); CHECK (inBuff.pos != cSize, "compressed data should be fully read") { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize); @@ -1427,7 +1421,7 @@ _output_error: /* fuzzing ZSTDMT_* interface */ -static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, +static int fuzzerTests_MT(U32 seed, int nbTests, int startTest, double compressibility, int bigTests) { const U32 maxSrcLog = bigTests ? 24 : 22; @@ -1441,9 +1435,9 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, size_t const dstBufferSize = srcBufferSize; BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize); U32 result = 0; - U32 testNb = 0; + int testNb = 0; U32 coreSeed = seed; - U32 nbThreads = 2; + int nbThreads = 2; ZSTDMT_CCtx* zc = ZSTDMT_createCCtx(nbThreads); /* will be reset sometimes */ ZSTD_DStream* zd = ZSTD_createDStream(); /* will be reset sometimes */ ZSTD_DStream* const zd_noise = ZSTD_createDStream(); @@ -1471,7 +1465,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */ memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */ ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */ - DISPLAYLEVEL(6, "Creating initial context with %u threads \n", nbThreads); + DISPLAYLEVEL(6, "Creating initial context with %i threads \n", nbThreads); /* catch up testNb */ for (testNb=1; testNb < startTest; testNb++) @@ -1549,13 +1543,13 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize; ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize); DISPLAYLEVEL(5, "Init with windowLog = %u, pledgedSrcSize = %u, dictSize = %u \n", - params.cParams.windowLog, (U32)pledgedSrcSize, (U32)dictSize); + params.cParams.windowLog, (unsigned)pledgedSrcSize, (unsigned)dictSize); params.fParams.checksumFlag = FUZ_rand(&lseed) & 1; params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1; params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1; DISPLAYLEVEL(5, "checksumFlag : %u \n", params.fParams.checksumFlag); - CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapSectionLog, FUZ_rand(&lseed) % 12) ); - CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_jobSize, FUZ_rand(&lseed) % (2*maxTestSize+1)) ); /* custome job size */ + CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapLog, FUZ_rand(&lseed) % 12) ); + CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_jobSize, FUZ_rand(&lseed) % (2*maxTestSize+1)) ); /* custom job size */ CHECK_Z( ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) ); } } } @@ -1573,9 +1567,9 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 }; outBuff.size = outBuff.pos + dstBuffSize; - DISPLAYLEVEL(6, "Sending %u bytes to compress \n", (U32)srcSize); + DISPLAYLEVEL(6, "Sending %u bytes to compress \n", (unsigned)srcSize); CHECK_Z( ZSTDMT_compressStream(zc, &outBuff, &inBuff) ); - DISPLAYLEVEL(6, "%u bytes read by ZSTDMT_compressStream \n", (U32)inBuff.pos); + DISPLAYLEVEL(6, "%u bytes read by ZSTDMT_compressStream \n", (unsigned)inBuff.pos); XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos); memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos); @@ -1588,10 +1582,10 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize); size_t const previousPos = outBuff.pos; outBuff.size = outBuff.pos + adjustedDstSize; - DISPLAYLEVEL(5, "Flushing into dst buffer of size %u \n", (U32)adjustedDstSize); + DISPLAYLEVEL(5, "Flushing into dst buffer of size %u \n", (unsigned)adjustedDstSize); CHECK_Z( ZSTDMT_flushStream(zc, &outBuff) ); assert(outBuff.pos >= previousPos); - DISPLAYLEVEL(6, "%u bytes flushed by ZSTDMT_flushStream \n", (U32)(outBuff.pos-previousPos)); + DISPLAYLEVEL(6, "%u bytes flushed by ZSTDMT_flushStream \n", (unsigned)(outBuff.pos-previousPos)); } } /* final frame epilogue */ @@ -1601,17 +1595,17 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize); size_t const previousPos = outBuff.pos; outBuff.size = outBuff.pos + adjustedDstSize; - DISPLAYLEVEL(5, "Ending into dst buffer of size %u \n", (U32)adjustedDstSize); + DISPLAYLEVEL(5, "Ending into dst buffer of size %u \n", (unsigned)adjustedDstSize); remainingToFlush = ZSTDMT_endStream(zc, &outBuff); CHECK (ZSTD_isError(remainingToFlush), "ZSTDMT_endStream error : %s", ZSTD_getErrorName(remainingToFlush)); assert(outBuff.pos >= previousPos); - DISPLAYLEVEL(6, "%u bytes flushed by ZSTDMT_endStream \n", (U32)(outBuff.pos-previousPos)); - DISPLAYLEVEL(5, "endStream : remainingToFlush : %u \n", (U32)remainingToFlush); + DISPLAYLEVEL(6, "%u bytes flushed by ZSTDMT_endStream \n", (unsigned)(outBuff.pos-previousPos)); + DISPLAYLEVEL(5, "endStream : remainingToFlush : %u \n", (unsigned)remainingToFlush); } } crcOrig = XXH64_digest(&xxhState); cSize = outBuff.pos; DISPLAYLEVEL(5, "Frame completed : %u bytes compressed into %u bytes \n", - (U32)totalTestSize, (U32)cSize); + (unsigned)totalTestSize, (unsigned)cSize); } /* multi - fragments decompression test */ @@ -1632,7 +1626,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, inBuff.size = inBuff.pos + readCSrcSize; outBuff.size = outBuff.pos + dstBuffSize; DISPLAYLEVEL(6, "ZSTD_decompressStream input %u bytes into outBuff %u bytes \n", - (U32)readCSrcSize, (U32)dstBuffSize); + (unsigned)readCSrcSize, (unsigned)dstBuffSize); decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff); if (ZSTD_isError(decompressionResult)) { DISPLAY("ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(decompressionResult)); @@ -1640,10 +1634,14 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, } CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult)); DISPLAYLEVEL(6, "total ingested (inBuff.pos) = %u and produced (outBuff.pos) = %u \n", - (U32)inBuff.pos, (U32)outBuff.pos); + (unsigned)inBuff.pos, (unsigned)outBuff.pos); } - CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (U32)outBuff.pos, (U32)totalTestSize); - CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (U32)inBuff.pos, (U32)cSize); + CHECK (outBuff.pos != totalTestSize, + "decompressed data : wrong size (%u != %u)", + (unsigned)outBuff.pos, (unsigned)totalTestSize ); + CHECK (inBuff.pos != cSize, + "compressed data should be fully read (%u != %u)", + (unsigned)inBuff.pos, (unsigned)cSize ); { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize); CHECK (crcDest!=crcOrig, "decompressed data corrupted"); @@ -1712,7 +1710,7 @@ static size_t setCCtxParameter(ZSTD_CCtx* zc, ZSTD_CCtx_params* cctxParams, } /* Tests for ZSTD_compress_generic() API */ -static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, +static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest, double compressibility, int bigTests) { U32 const maxSrcLog = bigTests ? 24 : 22; @@ -1726,7 +1724,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, size_t const dstBufferSize = srcBufferSize; BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize); U32 result = 0; - U32 testNb = 0; + int testNb = 0; U32 coreSeed = seed; ZSTD_CCtx* zc = ZSTD_createCCtx(); /* will be reset sometimes */ ZSTD_DStream* zd = ZSTD_createDStream(); /* will be reset sometimes */ @@ -1824,7 +1822,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1; { int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1; DISPLAYLEVEL(5, "t%u : compression level : %i \n", testNb, compressionLevel); - CHECK_Z (setCCtxParameter(zc, cctxParams, ZSTD_p_compressionLevel, compressionLevel, opaqueAPI) ); + CHECK_Z (setCCtxParameter(zc, cctxParams, ZSTD_c_compressionLevel, compressionLevel, opaqueAPI) ); } } else { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; @@ -1833,10 +1831,10 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, (ZSTD_maxCLevel() - (MAX(testLog, dictLog) / 2))) + 1; - U32 const cLevel = MIN(cLevelCandidate, cLevelMax); - DISPLAYLEVEL(5, "t%u: base cLevel : %u \n", testNb, cLevel); + int const cLevel = MIN(cLevelCandidate, cLevelMax); + DISPLAYLEVEL(5, "t%i: base cLevel : %u \n", testNb, cLevel); maxTestSize = FUZ_rLogLength(&lseed, testLog); - DISPLAYLEVEL(5, "t%u: maxTestSize : %u \n", testNb, (U32)maxTestSize); + DISPLAYLEVEL(5, "t%i: maxTestSize : %u \n", testNb, (unsigned)maxTestSize); oldTestLog = testLog; /* random dictionary selection */ dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0; @@ -1858,47 +1856,47 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1; cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1; cParams.searchLog = MIN(searchLogMax, cParams.searchLog); - cParams.searchLength += (FUZ_rand(&lseed) & 3) - 1; + cParams.minMatch += (FUZ_rand(&lseed) & 3) - 1; cParams.targetLength = (U32)((cParams.targetLength + 1 ) * (0.5 + ((double)(FUZ_rand(&lseed) & 127) / 128))); cParams = ZSTD_adjustCParams(cParams, pledgedSrcSize, dictSize); if (FUZ_rand(&lseed) & 1) { DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog); - CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_windowLog, cParams.windowLog, opaqueAPI) ); + CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_windowLog, cParams.windowLog, opaqueAPI) ); assert(cParams.windowLog >= ZSTD_WINDOWLOG_MIN); /* guaranteed by ZSTD_adjustCParams() */ windowLogMalus = (cParams.windowLog - ZSTD_WINDOWLOG_MIN) / 5; } if (FUZ_rand(&lseed) & 1) { DISPLAYLEVEL(5, "t%u: hashLog : %u \n", testNb, cParams.hashLog); - CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_hashLog, cParams.hashLog, opaqueAPI) ); + CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_hashLog, cParams.hashLog, opaqueAPI) ); } if (FUZ_rand(&lseed) & 1) { DISPLAYLEVEL(5, "t%u: chainLog : %u \n", testNb, cParams.chainLog); - CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_chainLog, cParams.chainLog, opaqueAPI) ); + CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_chainLog, cParams.chainLog, opaqueAPI) ); } - if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_searchLog, cParams.searchLog, opaqueAPI) ); - if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_minMatch, cParams.searchLength, opaqueAPI) ); - if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_targetLength, cParams.targetLength, opaqueAPI) ); + if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_searchLog, cParams.searchLog, opaqueAPI) ); + if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_minMatch, cParams.minMatch, opaqueAPI) ); + if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_targetLength, cParams.targetLength, opaqueAPI) ); /* mess with long distance matching parameters */ if (bigTests) { - if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, opaqueAPI) ); - if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), opaqueAPI) ); - if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), opaqueAPI) ); - if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_LDM_BUCKETSIZELOG_MAX), opaqueAPI) ); - if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashEveryLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN), opaqueAPI) ); + if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, opaqueAPI) ); + if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), opaqueAPI) ); + if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), opaqueAPI) ); + if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_BUCKETSIZELOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX), opaqueAPI) ); + if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashRateLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_HASHRATELOG_MIN, ZSTD_LDM_HASHRATELOG_MAX), opaqueAPI) ); } /* mess with frame parameters */ if (FUZ_rand(&lseed) & 1) { - U32 const checksumFlag = FUZ_rand(&lseed) & 1; + int const checksumFlag = FUZ_rand(&lseed) & 1; DISPLAYLEVEL(5, "t%u: frame checksum : %u \n", testNb, checksumFlag); - CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_checksumFlag, checksumFlag, opaqueAPI) ); + CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_checksumFlag, checksumFlag, opaqueAPI) ); } - if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_dictIDFlag, FUZ_rand(&lseed) & 1, opaqueAPI) ); - if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_contentSizeFlag, FUZ_rand(&lseed) & 1, opaqueAPI) ); + if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_dictIDFlag, FUZ_rand(&lseed) & 1, opaqueAPI) ); + if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_contentSizeFlag, FUZ_rand(&lseed) & 1, opaqueAPI) ); if (FUZ_rand(&lseed) & 1) { - DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (U32)pledgedSrcSize); + DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (unsigned)pledgedSrcSize); CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) ); } @@ -1906,17 +1904,19 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, if (bigTests || (FUZ_rand(&lseed) & 0xF) == 0xF) { U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1; U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1; - U32 const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax); - DISPLAYLEVEL(5, "t%u: nbThreads : %u \n", testNb, nbThreads); - CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_nbWorkers, nbThreads, opaqueAPI) ); + int const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax); + DISPLAYLEVEL(5, "t%i: nbThreads : %u \n", testNb, nbThreads); + CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_nbWorkers, nbThreads, opaqueAPI) ); if (nbThreads > 1) { U32 const jobLog = FUZ_rand(&lseed) % (testLog+1); - CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_overlapSizeLog, FUZ_rand(&lseed) % 10, opaqueAPI) ); - CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_jobSize, (U32)FUZ_rLogLength(&lseed, jobLog), opaqueAPI) ); + CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_overlapLog, FUZ_rand(&lseed) % 10, opaqueAPI) ); + CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_jobSize, (U32)FUZ_rLogLength(&lseed, jobLog), opaqueAPI) ); } } + /* Enable rsyncable mode 1 in 4 times. */ + setCCtxParameter(zc, cctxParams, ZSTD_c_rsyncable, (FUZ_rand(&lseed) % 4 == 0), opaqueAPI); - if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_forceMaxWindow, FUZ_rand(&lseed) & 1, opaqueAPI) ); + if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_forceMaxWindow, FUZ_rand(&lseed) & 1, opaqueAPI) ); /* Apply parameters */ if (opaqueAPI) { @@ -1936,7 +1936,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, size_t const setError = ZSTD_CCtx_setParametersUsingCCtxParams(zc, cctxParams); CHECK(!ZSTD_isError(setError), "ZSTD_CCtx_setParametersUsingCCtxParams should have failed"); } else { - size_t const setError = ZSTD_CCtx_setParameter(zc, ZSTD_p_windowLog, cParams.windowLog-1); + size_t const setError = ZSTD_CCtx_setParameter(zc, ZSTD_c_windowLog, cParams.windowLog-1); CHECK(!ZSTD_isError(setError), "ZSTD_CCtx_setParameter should have failed"); } } @@ -1961,9 +1961,9 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 }; outBuff.size = outBuff.pos + dstBuffSize; - CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, flush) ); + CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, flush) ); DISPLAYLEVEL(6, "t%u: compress consumed %u bytes (total : %u) ; flush: %u (total : %u) \n", - testNb, (U32)inBuff.pos, (U32)(totalTestSize + inBuff.pos), (U32)flush, (U32)outBuff.pos); + testNb, (unsigned)inBuff.pos, (unsigned)(totalTestSize + inBuff.pos), (unsigned)flush, (unsigned)outBuff.pos); XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos); memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos); @@ -1977,11 +1977,11 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1); size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize); outBuff.size = outBuff.pos + adjustedDstSize; - DISPLAYLEVEL(6, "t%u: End-flush into dst buffer of size %u \n", testNb, (U32)adjustedDstSize); - remainingToFlush = ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end); - DISPLAYLEVEL(6, "t%u: Total flushed so far : %u bytes \n", testNb, (U32)outBuff.pos); + DISPLAYLEVEL(6, "t%u: End-flush into dst buffer of size %u \n", testNb, (unsigned)adjustedDstSize); + remainingToFlush = ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end); + DISPLAYLEVEL(6, "t%u: Total flushed so far : %u bytes \n", testNb, (unsigned)outBuff.pos); CHECK( ZSTD_isError(remainingToFlush), - "ZSTD_compress_generic w/ ZSTD_e_end error : %s", + "ZSTD_compressStream2 w/ ZSTD_e_end error : %s", ZSTD_getErrorName(remainingToFlush) ); } } crcOrig = XXH64_digest(&xxhState); @@ -1993,7 +1993,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, /* multi - fragments decompression test */ if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) { - DISPLAYLEVEL(5, "resetting DCtx (dict:%08X) \n", (U32)(size_t)dict); + DISPLAYLEVEL(5, "resetting DCtx (dict:%p) \n", dict); CHECK_Z( ZSTD_resetDStream(zd) ); } else { if (dictSize) @@ -2010,19 +2010,19 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, inBuff.size = inBuff.pos + readCSrcSize; outBuff.size = outBuff.pos + dstBuffSize; DISPLAYLEVEL(6, "decompression presented %u new bytes (pos:%u/%u)\n", - (U32)readCSrcSize, (U32)inBuff.pos, (U32)cSize); + (unsigned)readCSrcSize, (unsigned)inBuff.pos, (unsigned)cSize); decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff); DISPLAYLEVEL(6, "so far: consumed = %u, produced = %u \n", - (U32)inBuff.pos, (U32)outBuff.pos); + (unsigned)inBuff.pos, (unsigned)outBuff.pos); if (ZSTD_isError(decompressionResult)) { DISPLAY("ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(decompressionResult)); findDiff(copyBuffer, dstBuffer, totalTestSize); } CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult)); - CHECK (inBuff.pos > cSize, "ZSTD_decompressStream consumes too much input : %u > %u ", (U32)inBuff.pos, (U32)cSize); + CHECK (inBuff.pos > cSize, "ZSTD_decompressStream consumes too much input : %u > %u ", (unsigned)inBuff.pos, (unsigned)cSize); } - CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (U32)inBuff.pos, (U32)cSize); - CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (U32)outBuff.pos, (U32)totalTestSize); + CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (unsigned)inBuff.pos, (unsigned)cSize); + CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (unsigned)outBuff.pos, (unsigned)totalTestSize); { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize); CHECK (crcDest!=crcOrig, "decompressed data corrupted"); @@ -2219,7 +2219,7 @@ int main(int argc, const char** argv) seed = h % 10000; } - DISPLAY("Seed = %u\n", seed); + DISPLAY("Seed = %u\n", (unsigned)seed); if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba); if (nbTests<=0) nbTests=1; |