diff options
Diffstat (limited to 'tests')
35 files changed, 2238 insertions, 581 deletions
diff --git a/tests/.gitignore b/tests/.gitignore index f408a7491213..b16e26e3d490 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -21,6 +21,7 @@ symbols legacy decodecorpus pool +poolTests invalidDictionaries # Tmp test directory @@ -48,6 +49,7 @@ grillResults.txt _* tmp* *.zst +*.gz result out diff --git a/tests/Makefile b/tests/Makefile index 3be79c159057..651833bbbaae 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,13 +1,11 @@ -# ########################################################################## -# Copyright (c) 2016-present, Yann Collet, Facebook, Inc. +# ################################################################ +# Copyright (c) 2015-present, Yann Collet, Facebook, Inc. # All rights reserved. # -# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. An additional grant -# of patent rights can be found in the PATENTS file in the same directory. -# ########################################################################## +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ################################################################ # datagen : Synthetic and parametrable data generator, for tests # fullbench : Precisely measure speed for each zstd inner functions # fullbench32: Same as fullbench, but forced to compile in 32-bits mode @@ -25,18 +23,18 @@ PRGDIR = ../programs PYTHON ?= python3 TESTARTEFACT := versionsTest namespaceTest -DEBUGLEVEL= 1 -DEBUGFLAGS= -g -DZSTD_DEBUG=$(DEBUGLEVEL) -CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ - -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) -CFLAGS ?= -O3 -CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ - -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ - -Wstrict-prototypes -Wundef -Wformat-security \ - -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ - -Wredundant-decls -CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) +DEBUGLEVEL ?= 1 +DEBUGFLAGS = -g -DZSTD_DEBUG=$(DEBUGLEVEL) +CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ + -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) +CFLAGS ?= -O3 +CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ + -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ + -Wstrict-prototypes -Wundef -Wformat-security \ + -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ + -Wredundant-decls +CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) +FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c @@ -46,10 +44,6 @@ ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) ZBUFF_FILES := $(ZSTDDIR)/deprecated/*.c ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c -ZSTD_OBJ := $(patsubst %.c,%.o, $(wildcard $(ZSTD_FILES)) ) -ZBUFF_OBJ := $(patsubst %.c,%.o, $(wildcard $(ZBUFF_FILES)) ) -ZDICT_OBJ := $(patsubst %.c,%.o, $(wildcard $(ZDICT_FILES)) ) - # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) @@ -64,8 +58,8 @@ endif MULTITHREAD = $(MULTITHREAD_CPP) $(MULTITHREAD_LD) VOID = /dev/null -ZSTREAM_TESTTIME ?= -T2mn -FUZZERTEST ?= -T5mn +ZSTREAM_TESTTIME ?= -T90s +FUZZERTEST ?= -T200s ZSTDRTTEST = --test-large-data DECODECORPUS_TESTTIME ?= -T30 @@ -77,7 +71,7 @@ all: fullbench fuzzer zstreamtest paramgrill datagen decodecorpus all32: fullbench32 fuzzer32 zstreamtest32 -allnothread: fullbench fuzzer paramgrill datagen decodecorpus +allnothread: fullbench fuzzer paramgrill datagen decodecorpus dll: fuzzer-dll zstreamtest-dll @@ -91,7 +85,7 @@ zstd-nolegacy: $(MAKE) -C $(PRGDIR) $@ gzstd: - $(MAKE) -C $(PRGDIR) $@ + $(MAKE) -C $(PRGDIR) zstd HAVE_ZLIB=1 fullbench32: CPPFLAGS += -m32 fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP) @@ -167,7 +161,7 @@ datagen : $(PRGDIR)/datagen.c datagencli.c $(CC) $(FLAGS) $^ -o $@$(EXT) roundTripCrash : $(ZSTD_FILES) roundTripCrash.c - $(CC) $(FLAGS) $^ -o $@$(EXT) + $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT) longmatch : $(ZSTD_FILES) longmatch.c $(CC) $(FLAGS) $^ -o $@$(EXT) @@ -192,8 +186,8 @@ else $(CC) $(FLAGS) $^ -o $@$(EXT) -Wl,-rpath=$(ZSTDDIR) $(ZSTDDIR)/libzstd.so endif -poolTests : poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c - $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT) +poolTests : poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/error_private.c + $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT) namespaceTest: if $(CC) namespaceTest.c ../lib/common/xxhash.c -o $@ ; then echo compilation should fail; exit 1 ; fi @@ -214,7 +208,7 @@ clean: zstreamtest$(EXT) zstreamtest32$(EXT) \ datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \ symbols$(EXT) invalidDictionaries$(EXT) legacy$(EXT) poolTests$(EXT) \ - decodecorpus$(EXT) + decodecorpus$(EXT) @echo Cleaning completed @@ -247,7 +241,7 @@ endif #----------------------------------------------------------------------------- -#make tests validated only for MSYS, Linux, OSX, BSD, Hurd and Solaris targets +# make tests validated only for below targets #----------------------------------------------------------------------------- ifneq (,$(filter $(HOST_OS),MSYS POSIX)) @@ -284,13 +278,13 @@ test-zstd-nolegacy: ZSTD = $(PRGDIR)/zstd-nolegacy test-zstd-nolegacy: zstd-nolegacy zstd-playTests test-gzstd: gzstd - $(PRGDIR)/zstd README.md test-zstd-speed.py - gzip README.md test-zstd-speed.py + $(PRGDIR)/zstd -f README.md test-zstd-speed.py + gzip -f README.md test-zstd-speed.py cat README.md.zst test-zstd-speed.py.gz >zstd_gz.zst cat README.md.gz test-zstd-speed.py.zst >gz_zstd.gz - $(PRGDIR)/zstd -d README.md.gz -o README2.md - $(PRGDIR)/zstd -d README.md.gz test-zstd-speed.py.gz - $(PRGDIR)/zstd -d zstd_gz.zst gz_zstd.gz + $(PRGDIR)/zstd -df README.md.gz -o README2.md + $(PRGDIR)/zstd -df README.md.gz test-zstd-speed.py.gz + $(PRGDIR)/zstd -df zstd_gz.zst gz_zstd.gz $(DIFF) -q zstd_gz gz_zstd echo Hello World ZSTD | $(PRGDIR)/zstd -c - >hello.zst echo Hello World GZIP | gzip -c - >hello.gz @@ -298,6 +292,7 @@ test-gzstd: gzstd cat hello.zst hello.gz hello.txt >hello_zst_gz_txt.gz $(PRGDIR)/zstd -dcf hello.* $(PRGDIR)/zstd -dcf - <hello_zst_gz_txt.gz + $(RM) *.gz *.zst README2.md gz_zstd zstd_gz hello.txt test-fullbench: fullbench datagen $(QEMU_SYS) ./fullbench -i1 @@ -321,8 +316,9 @@ test-zbuff32: zbufftest32 test-zstream: zstreamtest $(QEMU_SYS) ./zstreamtest $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS) - $(QEMU_SYS) ./zstreamtest --mt $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS) - $(QEMU_SYS) ./zstreamtest --newapi $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS) + $(QEMU_SYS) ./zstreamtest --mt -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS) + $(QEMU_SYS) ./zstreamtest --newapi -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS) + $(QEMU_SYS) ./zstreamtest --opaqueapi -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS) test-zstream32: zstreamtest32 $(QEMU_SYS) ./zstreamtest32 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS) @@ -378,4 +374,37 @@ test-decodecorpus-cli: decodecorpus test-pool: poolTests $(QEMU_SYS) ./poolTests +test-lz4: ZSTD = LD_LIBRARY_PATH=/usr/local/lib $(PRGDIR)/zstd +test-lz4: ZSTD_LZ4 = LD_LIBRARY_PATH=/usr/local/lib ./lz4 +test-lz4: ZSTD_UNLZ4 = LD_LIBRARY_PATH=/usr/local/lib ./unlz4 +test-lz4: zstd decodecorpus + ln -s $(PRGDIR)/zstd lz4 + ln -s $(PRGDIR)/zstd unlz4 + + ./decodecorpus -ptmp + # lz4 -> zstd + lz4 < tmp | \ + $(ZSTD) -d | \ + cmp - tmp + lz4 < tmp | \ + $(ZSTD_UNLZ4) | \ + cmp - tmp + # zstd -> lz4 + $(ZSTD) --format=lz4 < tmp | \ + lz4 -d | \ + cmp - tmp + $(ZSTD_LZ4) < tmp | \ + lz4 -d | \ + cmp - tmp + # zstd -> zstd + $(ZSTD) --format=lz4 < tmp | \ + $(ZSTD) -d | \ + cmp - tmp + # zstd -> zstd + $(ZSTD) < tmp | \ + $(ZSTD) -d | \ + cmp - tmp + + rm tmp lz4 unlz4 + endif diff --git a/tests/datagencli.c b/tests/datagencli.c index bf9601f20976..4814974e27f5 100644 --- a/tests/datagencli.c +++ b/tests/datagencli.c @@ -1,10 +1,11 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2015-present, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. */ diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 23166bd67f0a..e697e519cfc1 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1,10 +1,11 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2017-present, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. */ #include <limits.h> @@ -44,7 +45,7 @@ **************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static U32 g_displayLevel = 0; +static U32 g_displayLevel = 2; #define DISPLAYUPDATE(...) \ do { \ @@ -173,20 +174,19 @@ const char *BLOCK_TYPES[] = {"raw", "rle", "compressed"}; #define MAX_DECOMPRESSED_SIZE (1ULL << MAX_DECOMPRESSED_SIZE_LOG) #define MAX_WINDOW_LOG 22 /* Recommended support is 8MB, so limit to 4MB + mantissa */ -#define MAX_BLOCK_SIZE (128ULL * 1024) #define MIN_SEQ_LEN (3) -#define MAX_NB_SEQ ((MAX_BLOCK_SIZE + MIN_SEQ_LEN - 1) / MIN_SEQ_LEN) +#define MAX_NB_SEQ ((ZSTD_BLOCKSIZE_MAX + MIN_SEQ_LEN - 1) / MIN_SEQ_LEN) BYTE CONTENT_BUFFER[MAX_DECOMPRESSED_SIZE]; BYTE FRAME_BUFFER[MAX_DECOMPRESSED_SIZE * 2]; -BYTE LITERAL_BUFFER[MAX_BLOCK_SIZE]; +BYTE LITERAL_BUFFER[ZSTD_BLOCKSIZE_MAX]; seqDef SEQUENCE_BUFFER[MAX_NB_SEQ]; -BYTE SEQUENCE_LITERAL_BUFFER[MAX_BLOCK_SIZE]; /* storeSeq expects a place to copy literals to */ -BYTE SEQUENCE_LLCODE[MAX_BLOCK_SIZE]; -BYTE SEQUENCE_MLCODE[MAX_BLOCK_SIZE]; -BYTE SEQUENCE_OFCODE[MAX_BLOCK_SIZE]; +BYTE SEQUENCE_LITERAL_BUFFER[ZSTD_BLOCKSIZE_MAX]; /* storeSeq expects a place to copy literals to */ +BYTE SEQUENCE_LLCODE[ZSTD_BLOCKSIZE_MAX]; +BYTE SEQUENCE_MLCODE[ZSTD_BLOCKSIZE_MAX]; +BYTE SEQUENCE_OFCODE[ZSTD_BLOCKSIZE_MAX]; unsigned WKSP[1024]; @@ -237,6 +237,18 @@ typedef struct { size_t dictContentSize; BYTE* dictContent; } dictInfo; + +typedef enum { + gt_frame = 0, /* generate frames */ + gt_block, /* generate compressed blocks without block/frame headers */ +} genType_e; + +/*-******************************************************* +* Global variables (set from command line) +*********************************************************/ +U32 g_maxDecompressedSizeLog = MAX_DECOMPRESSED_SIZE_LOG; /* <= 20 */ +U32 g_maxBlockSize = ZSTD_BLOCKSIZE_MAX; /* <= 128 KB */ + /*-******************************************************* * Generator Functions *********************************************************/ @@ -273,12 +285,12 @@ static void writeFrameHeader(U32* seed, frame_t* frame, dictInfo info) { /* Generate random content size */ size_t highBit; - if (RAND(seed) & 7) { + if (RAND(seed) & 7 && g_maxDecompressedSizeLog > 7) { /* do content of at least 128 bytes */ - highBit = 1ULL << RAND_range(seed, 7, MAX_DECOMPRESSED_SIZE_LOG); + highBit = 1ULL << RAND_range(seed, 7, g_maxDecompressedSizeLog); } else if (RAND(seed) & 3) { /* do small content */ - highBit = 1ULL << RAND_range(seed, 0, 7); + highBit = 1ULL << RAND_range(seed, 0, MIN(7, 1U << g_maxDecompressedSizeLog)); } else { /* 0 size frame */ highBit = 0; @@ -342,10 +354,10 @@ static void writeFrameHeader(U32* seed, frame_t* frame, dictInfo info) } } - DISPLAYLEVEL(2, " frame content size:\t%u\n", (U32)fh.contentSize); - DISPLAYLEVEL(2, " frame window size:\t%u\n", fh.windowSize); - DISPLAYLEVEL(2, " content size flag:\t%d\n", contentSizeFlag); - DISPLAYLEVEL(2, " single segment flag:\t%d\n", singleSegment); + DISPLAYLEVEL(3, " frame content size:\t%u\n", (U32)fh.contentSize); + DISPLAYLEVEL(3, " frame window size:\t%u\n", fh.windowSize); + DISPLAYLEVEL(3, " content size flag:\t%d\n", contentSizeFlag); + DISPLAYLEVEL(3, " single segment flag:\t%d\n", singleSegment); frame->data = op + pos; frame->header = fh; @@ -358,7 +370,7 @@ static size_t writeLiteralsBlockSimple(U32* seed, frame_t* frame, size_t content int const type = RAND(seed) % 2; int const sizeFormatDesc = RAND(seed) % 8; size_t litSize; - size_t maxLitSize = MIN(contentSize, MAX_BLOCK_SIZE); + size_t maxLitSize = MIN(contentSize, g_maxBlockSize); if (sizeFormatDesc == 0) { /* Size_FormatDesc = ?0 */ @@ -452,7 +464,7 @@ static size_t writeHufHeader(U32* seed, HUF_CElt* hufTable, void* dst, size_t ds return op - ostart; } -/* Write a Huffman coded literals block and return the litearls size */ +/* Write a Huffman coded literals block and return the literals size */ static size_t writeLiteralsBlockCompressed(U32* seed, frame_t* frame, size_t contentSize) { BYTE* origop = (BYTE*)frame->data; @@ -463,7 +475,7 @@ static size_t writeLiteralsBlockCompressed(U32* seed, frame_t* frame, size_t con size_t litSize; size_t hufHeaderSize = 0; size_t compressedSize = 0; - size_t maxLitSize = MIN(contentSize-3, MAX_BLOCK_SIZE); + size_t maxLitSize = MIN(contentSize-3, g_maxBlockSize); symbolEncodingType_e hType; @@ -867,7 +879,7 @@ static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr, frame->stats.offsetSymbolSet, 28)) { Offtype = set_repeat; } else if (!(RAND(seed) & 3)) { - FSE_buildCTable_wksp(CTable_OffsetBits, OF_defaultNorm, MaxOff, OF_defaultNormLog, scratchBuffer, sizeof(scratchBuffer)); + FSE_buildCTable_wksp(CTable_OffsetBits, OF_defaultNorm, DefaultMaxOff, OF_defaultNormLog, scratchBuffer, sizeof(scratchBuffer)); Offtype = set_basic; } else { size_t nbSeq_1 = nbSeq; @@ -1020,9 +1032,9 @@ static void writeBlock(U32* seed, frame_t* frame, size_t contentSize, BYTE *const header = (BYTE*)frame->data; BYTE *op = header + 3; - DISPLAYLEVEL(3, " block:\n"); - DISPLAYLEVEL(3, " block content size: %u\n", (U32)contentSize); - DISPLAYLEVEL(3, " last block: %s\n", lastBlock ? "yes" : "no"); + DISPLAYLEVEL(4, " block:\n"); + DISPLAYLEVEL(4, " block content size: %u\n", (U32)contentSize); + DISPLAYLEVEL(4, " last block: %s\n", lastBlock ? "yes" : "no"); if (blockTypeDesc == 0) { /* Raw data frame */ @@ -1052,7 +1064,7 @@ static void writeBlock(U32* seed, frame_t* frame, size_t contentSize, frame->data = op; compressedSize = writeCompressedBlock(seed, frame, contentSize, info); - if (compressedSize > contentSize) { + if (compressedSize >= contentSize) { /* compressed block must be strictly smaller than uncompressed one */ blockType = 0; memcpy(op, frame->src, contentSize); @@ -1068,8 +1080,8 @@ static void writeBlock(U32* seed, frame_t* frame, size_t contentSize, } frame->src = (BYTE*)frame->src + contentSize; - DISPLAYLEVEL(3, " block type: %s\n", BLOCK_TYPES[blockType]); - DISPLAYLEVEL(3, " block size field: %u\n", (U32)blockSize); + DISPLAYLEVEL(4, " block type: %s\n", BLOCK_TYPES[blockType]); + DISPLAYLEVEL(4, " block size field: %u\n", (U32)blockSize); header[0] = (BYTE) ((lastBlock | (blockType << 1) | (blockSize << 3)) & 0xff); MEM_writeLE16(header + 1, (U16) (blockSize >> 5)); @@ -1080,7 +1092,7 @@ static void writeBlock(U32* seed, frame_t* frame, size_t contentSize, static void writeBlocks(U32* seed, frame_t* frame, dictInfo info) { size_t contentLeft = frame->header.contentSize; - size_t const maxBlockSize = MIN(MAX_BLOCK_SIZE, frame->header.windowSize); + size_t const maxBlockSize = MIN(g_maxBlockSize, frame->header.windowSize); while (1) { /* 1 in 4 chance of ending frame */ int const lastBlock = contentLeft > maxBlockSize ? 0 : !(RAND(seed) & 3); @@ -1089,13 +1101,13 @@ static void writeBlocks(U32* seed, frame_t* frame, dictInfo info) blockContentSize = contentLeft; } else { if (contentLeft > 0 && (RAND(seed) & 7)) { - /* some variable size blocks */ + /* some variable size block */ blockContentSize = RAND(seed) % (MIN(maxBlockSize, contentLeft)+1); } else if (contentLeft > maxBlockSize && (RAND(seed) & 1)) { - /* some full size blocks */ + /* some full size block */ blockContentSize = maxBlockSize; } else { - /* some empty blocks */ + /* some empty block */ blockContentSize = 0; } } @@ -1111,7 +1123,7 @@ static void writeChecksum(frame_t* frame) { /* write checksum so implementations can verify their output */ U64 digest = XXH64(frame->srcStart, (BYTE*)frame->src-(BYTE*)frame->srcStart, 0); - DISPLAYLEVEL(2, " checksum: %08x\n", (U32)digest); + DISPLAYLEVEL(3, " checksum: %08x\n", (U32)digest); MEM_writeLE32(frame->data, (U32)digest); frame->data = (BYTE*)frame->data + 4; } @@ -1132,8 +1144,7 @@ static void outputBuffer(const void* buf, size_t size, const char* const path) exit(1); } - { - size_t fsize = size; + { size_t fsize = size; size_t written = 0; while (written < fsize) { written += fwrite(ip + written, 1, fsize - written, out); @@ -1164,11 +1175,64 @@ static void initFrame(frame_t* fr) fr->stats.rep[2] = 8; } +/** + * Generated a single zstd compressed block with no block/frame header. + * Returns the final seed. + */ +static U32 generateCompressedBlock(U32 seed, frame_t* frame, dictInfo info) +{ + size_t blockContentSize; + int blockWritten = 0; + BYTE* op; + DISPLAYLEVEL(4, "block seed: %u\n", seed); + initFrame(frame); + op = (BYTE*)frame->data; + + while (!blockWritten) { + size_t cSize; + /* generate window size */ + { int const exponent = RAND(&seed) % (MAX_WINDOW_LOG - 10); + int const mantissa = RAND(&seed) % 8; + frame->header.windowSize = (1U << (exponent + 10)); + frame->header.windowSize += (frame->header.windowSize / 8) * mantissa; + } + + /* generate content size */ + { size_t const maxBlockSize = MIN(g_maxBlockSize, frame->header.windowSize); + if (RAND(&seed) & 15) { + /* some full size blocks */ + blockContentSize = maxBlockSize; + } else if (RAND(&seed) & 7 && g_maxBlockSize >= (1U << 7)) { + /* some small blocks <= 128 bytes*/ + blockContentSize = RAND(&seed) % (1U << 7); + } else { + /* some variable size blocks */ + blockContentSize = RAND(&seed) % maxBlockSize; + } + } + + /* try generating a compressed block */ + frame->oldStats = frame->stats; + frame->data = op; + cSize = writeCompressedBlock(&seed, frame, blockContentSize, info); + if (cSize >= blockContentSize) { /* compressed size must be strictly smaller than decompressed size : https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#blocks */ + /* data doesn't compress -- try again */ + frame->stats = frame->oldStats; /* don't update the stats */ + DISPLAYLEVEL(5, " can't compress block : try again \n"); + } else { + blockWritten = 1; + DISPLAYLEVEL(4, " block size: %u \n", (U32)cSize); + frame->src = (BYTE*)frame->src + blockContentSize; + } + } + return seed; +} + /* Return the final seed */ static U32 generateFrame(U32 seed, frame_t* fr, dictInfo info) { /* generate a complete frame */ - DISPLAYLEVEL(1, "frame seed: %u\n", seed); + DISPLAYLEVEL(3, "frame seed: %u\n", seed); initFrame(fr); writeFrameHeader(&seed, fr, info); @@ -1182,7 +1246,8 @@ static U32 generateFrame(U32 seed, frame_t* fr, dictInfo info) * Dictionary Helper Functions *********************************************************/ /* returns 0 if successful, otherwise returns 1 upon error */ -static int genRandomDict(U32 dictID, U32 seed, size_t dictSize, BYTE* fullDict){ +static int genRandomDict(U32 dictID, U32 seed, size_t dictSize, BYTE* fullDict) +{ /* allocate space for samples */ int ret = 0; unsigned const numSamples = 4; @@ -1194,27 +1259,20 @@ static int genRandomDict(U32 dictID, U32 seed, size_t dictSize, BYTE* fullDict){ } /* generate samples */ - { - unsigned literalValue = 1; + { unsigned literalValue = 1; unsigned samplesPos = 0; size_t currSize = 1; while (literalValue <= 4) { sampleSizes[literalValue - 1] = currSize; - { - size_t k; + { size_t k; for (k = 0; k < currSize; k++) { *(samples + (samplesPos++)) = (BYTE)literalValue; - } - } + } } literalValue++; currSize *= 16; - } - } - + } } - { - /* create variables */ - size_t dictWriteSize = 0; + { size_t dictWriteSize = 0; ZDICT_params_t zdictParams; size_t const headerSize = MAX(dictSize/4, 256); size_t const dictContentSize = dictSize - headerSize; @@ -1322,7 +1380,7 @@ cleanup: return ret; } -static size_t testDecodeWithDict(U32 seed) +static size_t testDecodeWithDict(U32 seed, genType_e genType) { /* create variables */ size_t const dictSize = RAND(&seed) % (10 << 20) + ZDICT_DICTSIZE_MIN + ZDICT_CONTENTSIZE_MIN; @@ -1334,45 +1392,53 @@ static size_t testDecodeWithDict(U32 seed) } /* generate random dictionary */ - { - int const ret = genRandomDict(dictID, seed, dictSize, fullDict); - if (ret != 0) { - errorDetected = ERROR(GENERIC); - goto dictTestCleanup; - } + if (genRandomDict(dictID, seed, dictSize, fullDict)) { /* return 0 on success */ + errorDetected = ERROR(GENERIC); + goto dictTestCleanup; } - { - frame_t fr; + { frame_t fr; + dictInfo info; + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + size_t ret; - /* generate frame */ - { - size_t const headerSize = MAX(dictSize/4, 256); + /* get dict info */ + { size_t const headerSize = MAX(dictSize/4, 256); size_t const dictContentSize = dictSize-headerSize; BYTE* const dictContent = fullDict+headerSize; - dictInfo const info = initDictInfo(1, dictContentSize, dictContent, dictID); - seed = generateFrame(seed, &fr, info); + info = initDictInfo(1, dictContentSize, dictContent, dictID); } /* manually decompress and check difference */ - { - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - { - size_t const returnValue = ZSTD_decompress_usingDict(dctx, DECOMPRESSED_BUFFER, MAX_DECOMPRESSED_SIZE, - fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, - fullDict, dictSize); - if (ZSTD_isError(returnValue)) { - errorDetected = returnValue; - goto dictTestCleanup; - } - } - - if (memcmp(DECOMPRESSED_BUFFER, fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart) != 0) { - errorDetected = ERROR(corruption_detected); + if (genType == gt_frame) { + /* Test frame */ + generateFrame(seed, &fr, info); + ret = ZSTD_decompress_usingDict(dctx, DECOMPRESSED_BUFFER, MAX_DECOMPRESSED_SIZE, + fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, + fullDict, dictSize); + } else { + /* Test block */ + generateCompressedBlock(seed, &fr, info); + ret = ZSTD_decompressBegin_usingDict(dctx, fullDict, dictSize); + if (ZSTD_isError(ret)) { + errorDetected = ret; + ZSTD_freeDCtx(dctx); goto dictTestCleanup; } - ZSTD_freeDCtx(dctx); + ret = ZSTD_decompressBlock(dctx, DECOMPRESSED_BUFFER, MAX_DECOMPRESSED_SIZE, + fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart); + } + ZSTD_freeDCtx(dctx); + + if (ZSTD_isError(ret)) { + errorDetected = ret; + goto dictTestCleanup; + } + + if (memcmp(DECOMPRESSED_BUFFER, fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart) != 0) { + errorDetected = ERROR(corruption_detected); + goto dictTestCleanup; } } @@ -1381,7 +1447,87 @@ dictTestCleanup: return errorDetected; } -static int runTestMode(U32 seed, unsigned numFiles, unsigned const testDurationS) +static size_t testDecodeRawBlock(frame_t* fr) +{ + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + size_t ret = ZSTD_decompressBegin(dctx); + if (ZSTD_isError(ret)) return ret; + + ret = ZSTD_decompressBlock( + dctx, + DECOMPRESSED_BUFFER, MAX_DECOMPRESSED_SIZE, + fr->dataStart, (BYTE*)fr->data - (BYTE*)fr->dataStart); + ZSTD_freeDCtx(dctx); + if (ZSTD_isError(ret)) return ret; + + if (memcmp(DECOMPRESSED_BUFFER, fr->srcStart, + (BYTE*)fr->src - (BYTE*)fr->srcStart) != 0) { + return ERROR(corruption_detected); + } + + return ret; +} + +static int runBlockTest(U32* seed) +{ + frame_t fr; + U32 const seedCopy = *seed; + { dictInfo const info = initDictInfo(0, 0, NULL, 0); + *seed = generateCompressedBlock(*seed, &fr, info); + } + + { size_t const r = testDecodeRawBlock(&fr); + if (ZSTD_isError(r)) { + DISPLAY("Error in block mode on test seed %u: %s\n", seedCopy, + ZSTD_getErrorName(r)); + return 1; + } + } + + { size_t const r = testDecodeWithDict(*seed, gt_block); + if (ZSTD_isError(r)) { + DISPLAY("Error in block mode with dictionary on test seed %u: %s\n", + seedCopy, ZSTD_getErrorName(r)); + return 1; + } + } + return 0; +} + +static int runFrameTest(U32* seed) +{ + frame_t fr; + U32 const seedCopy = *seed; + { dictInfo const info = initDictInfo(0, 0, NULL, 0); + *seed = generateFrame(*seed, &fr, info); + } + + { size_t const r = testDecodeSimple(&fr); + if (ZSTD_isError(r)) { + DISPLAY("Error in simple mode on test seed %u: %s\n", + seedCopy, ZSTD_getErrorName(r)); + return 1; + } + } + { size_t const r = testDecodeStreaming(&fr); + if (ZSTD_isError(r)) { + DISPLAY("Error in streaming mode on test seed %u: %s\n", + seedCopy, ZSTD_getErrorName(r)); + return 1; + } + } + { size_t const r = testDecodeWithDict(*seed, gt_frame); /* avoid big dictionaries */ + if (ZSTD_isError(r)) { + DISPLAY("Error in dictionary mode on test seed %u: %s\n", + seedCopy, ZSTD_getErrorName(r)); + return 1; + } + } + return 0; +} + +static int runTestMode(U32 seed, unsigned numFiles, unsigned const testDurationS, + genType_e genType) { unsigned fnum; @@ -1393,39 +1539,15 @@ static int runTestMode(U32 seed, unsigned numFiles, unsigned const testDurationS DISPLAY("seed: %u\n", seed); for (fnum = 0; fnum < numFiles || clockSpan(startClock) < maxClockSpan; fnum++) { - frame_t fr; - U32 const seedCopy = seed; if (fnum < numFiles) DISPLAYUPDATE("\r%u/%u ", fnum, numFiles); else DISPLAYUPDATE("\r%u ", fnum); - { - dictInfo const info = initDictInfo(0, 0, NULL, 0); - seed = generateFrame(seed, &fr, info); - } - - { size_t const r = testDecodeSimple(&fr); - if (ZSTD_isError(r)) { - DISPLAY("Error in simple mode on test seed %u: %s\n", seedCopy, - ZSTD_getErrorName(r)); - return 1; - } - } - { size_t const r = testDecodeStreaming(&fr); - if (ZSTD_isError(r)) { - DISPLAY("Error in streaming mode on test seed %u: %s\n", seedCopy, - ZSTD_getErrorName(r)); - return 1; - } - } - { - /* don't create a dictionary that is too big */ - size_t const r = testDecodeWithDict(seed); - if (ZSTD_isError(r)) { - DISPLAY("Error in dictionary mode on test seed %u: %s\n", seedCopy, ZSTD_getErrorName(r)); - return 1; - } + { int const ret = (genType == gt_frame) ? + runFrameTest(&seed) : + runBlockTest(&seed); + if (ret) return ret; } } @@ -1440,17 +1562,19 @@ static int runTestMode(U32 seed, unsigned numFiles, unsigned const testDurationS *********************************************************/ static int generateFile(U32 seed, const char* const path, - const char* const origPath) + const char* const origPath, genType_e genType) { frame_t fr; DISPLAY("seed: %u\n", seed); - { - dictInfo const info = initDictInfo(0, 0, NULL, 0); - generateFrame(seed, &fr, info); + { dictInfo const info = initDictInfo(0, 0, NULL, 0); + if (genType == gt_frame) { + generateFrame(seed, &fr, info); + } else { + generateCompressedBlock(seed, &fr, info); + } } - outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, path); if (origPath) { outputBuffer(fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, origPath); @@ -1459,7 +1583,7 @@ static int generateFile(U32 seed, const char* const path, } static int generateCorpus(U32 seed, unsigned numFiles, const char* const path, - const char* const origPath) + const char* const origPath, genType_e genType) { char outPath[MAX_PATH]; unsigned fnum; @@ -1471,9 +1595,12 @@ static int generateCorpus(U32 seed, unsigned numFiles, const char* const path, DISPLAYUPDATE("\r%u/%u ", fnum, numFiles); - { - dictInfo const info = initDictInfo(0, 0, NULL, 0); - seed = generateFrame(seed, &fr, info); + { dictInfo const info = initDictInfo(0, 0, NULL, 0); + if (genType == gt_frame) { + seed = generateFrame(seed, &fr, info); + } else { + seed = generateCompressedBlock(seed, &fr, info); + } } if (snprintf(outPath, MAX_PATH, "%s/z%06u.zst", path, fnum) + 1 > MAX_PATH) { @@ -1497,7 +1624,8 @@ static int generateCorpus(U32 seed, unsigned numFiles, const char* const path, } static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const path, - const char* const origPath, const size_t dictSize) + const char* const origPath, const size_t dictSize, + genType_e genType) { char outPath[MAX_PATH]; BYTE* fullDict; @@ -1517,8 +1645,7 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const } /* randomly generate the dictionary */ - { - int const ret = genRandomDict(dictID, seed, dictSize, fullDict); + { int const ret = genRandomDict(dictID, seed, dictSize, fullDict); if (ret != 0) { errorDetected = ret; goto dictCleanup; @@ -1539,8 +1666,7 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const } /* generate random compressed/decompressed files */ - { - unsigned fnum; + { unsigned fnum; for (fnum = 0; fnum < MAX(numFiles, 1); fnum++) { frame_t fr; DISPLAYUPDATE("\r%u/%u ", fnum, numFiles); @@ -1549,7 +1675,11 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const size_t const dictContentSize = dictSize-headerSize; BYTE* const dictContent = fullDict+headerSize; dictInfo const info = initDictInfo(1, dictContentSize, dictContent, dictID); - seed = generateFrame(seed, &fr, info); + if (genType == gt_frame) { + seed = generateFrame(seed, &fr, info); + } else { + seed = generateCompressedBlock(seed, &fr, info); + } } if (numFiles != 0) { @@ -1626,9 +1756,13 @@ static void advancedUsage(const char* programName) { usage(programName); DISPLAY( "\n"); - DISPLAY( "Advanced arguments :\n"); - DISPLAY( " --content-size : always include the content size in the frame header\n"); - DISPLAY( " --use-dict=# : include a dictionary used to decompress the corpus\n"); + DISPLAY( "Advanced arguments :\n"); + DISPLAY( " --content-size : always include the content size in the frame header\n"); + DISPLAY( " --use-dict=# : include a dictionary used to decompress the corpus\n"); + DISPLAY( " --gen-blocks : generate raw compressed blocks without block/frame headers\n"); + DISPLAY( " --max-block-size-log=# : max block size log, must be in range [2, 17]\n"); + DISPLAY( " --max-content-size-log=# : max content size log, must be <= 20\n"); + DISPLAY( " (this is ignored with gen-blocks)\n"); } /*! readU32FromChar() : @@ -1675,6 +1809,7 @@ int main(int argc, char** argv) const char* origPath = NULL; int useDict = 0; unsigned dictSize = (10 << 10); /* 10 kB default */ + genType_e genType = gt_frame; int argNb; @@ -1738,6 +1873,17 @@ int main(int argc, char** argv) } else if (longCommandWArg(&argument, "use-dict=")) { dictSize = readU32FromChar(&argument); useDict = 1; + } else if (strcmp(argument, "gen-blocks") == 0) { + genType = gt_block; + } else if (longCommandWArg(&argument, "max-block-size-log=")) { + U32 value = readU32FromChar(&argument); + if (value >= 2 && value <= ZSTD_BLOCKSIZE_MAX) { + g_maxBlockSize = 1U << value; + } + } else if (longCommandWArg(&argument, "max-content-size-log=")) { + U32 value = readU32FromChar(&argument); + g_maxDecompressedSizeLog = + MIN(MAX_DECOMPRESSED_SIZE_LOG, value); } else { advancedUsage(argv[0]); return 1; @@ -1754,7 +1900,7 @@ int main(int argc, char** argv) } if (testMode) { - return runTestMode(seed, numFiles, testDuration); + return runTestMode(seed, numFiles, testDuration, genType); } else { if (testDuration) { DISPLAY("Error: -T requires test mode (-t)\n\n"); @@ -1770,12 +1916,12 @@ int main(int argc, char** argv) } if (numFiles == 0 && useDict == 0) { - return generateFile(seed, path, origPath); + return generateFile(seed, path, origPath, genType); } else if (useDict == 0){ - return generateCorpus(seed, numFiles, path, origPath); + return generateCorpus(seed, numFiles, path, origPath, genType); } else { /* should generate files with a dictionary */ - return generateCorpusWithDict(seed, numFiles, path, origPath, dictSize); + return generateCorpusWithDict(seed, numFiles, path, origPath, dictSize, genType); } } diff --git a/tests/fullbench.c b/tests/fullbench.c index 78a70940f9d2..db00ce217b0e 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -1,10 +1,11 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2015-present, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. */ @@ -375,6 +376,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) skippedSize = frameHeaderSize + ZSTD_blockHeaderSize; memcpy(buff2, dstBuff+skippedSize, g_cSize-skippedSize); srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */ + ZSTD_decompressBegin(g_zdc); break; } case 32: /* ZSTD_decodeSeqHeaders */ @@ -421,8 +423,6 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) { U32 loopNb; # define TIME_SEC_MICROSEC (1*1000000ULL) /* 1 second */ U64 const clockLoop = TIMELOOP_S * TIME_SEC_MICROSEC; - UTIL_freq_t ticksPerSecond; - UTIL_initTimer(&ticksPerSecond); DISPLAY("%2i- %-30.30s : \r", benchNb, benchName); for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) { UTIL_time_t clockStart; @@ -430,13 +430,13 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) U32 nbRounds; UTIL_sleepMilli(1); /* give processor time to other processes */ - UTIL_waitForNextTick(ticksPerSecond); - UTIL_getTime(&clockStart); - for (nbRounds=0; UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop; nbRounds++) { + UTIL_waitForNextTick(); + clockStart = UTIL_getTime(); + for (nbRounds=0; UTIL_clockSpanMicro(clockStart) < clockLoop; nbRounds++) { benchResult = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize); if (ZSTD_isError(benchResult)) { DISPLAY("ERROR ! %s() => %s !! \n", benchName, ZSTD_getErrorName(benchResult)); exit(1); } } - { U64 const clockSpanMicro = UTIL_clockSpanMicro(clockStart, ticksPerSecond); + { U64 const clockSpanMicro = UTIL_clockSpanMicro(clockStart); double const averageTime = (double)clockSpanMicro / TIME_SEC_MICROSEC / nbRounds; if (averageTime < bestTime) bestTime = averageTime; DISPLAY("%2i- %-30.30s : %7.1f MB/s (%9u)\r", loopNb, benchName, (double)srcSize / (1 MB) / bestTime, (U32)benchResult); diff --git a/tests/fuzz/Makefile b/tests/fuzz/Makefile index da22ed0d0baa..d9b00fd2af79 100644 --- a/tests/fuzz/Makefile +++ b/tests/fuzz/Makefile @@ -1,97 +1,97 @@ -# ########################################################################## +# ################################################################ # Copyright (c) 2016-present, Facebook, Inc. # All rights reserved. # -# This Makefile is validated for Linux, and macOS targets -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. An additional grant -# of patent rights can be found in the PATENTS file in the same directory. -# ########################################################################## +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ################################################################ +# Optionally user defined flags CFLAGS ?= -O3 CXXFLAGS ?= -O3 +CPPFLAGS ?= +LDFLAGS ?= +ARFLAGS ?= +LIB_FUZZING_ENGINE ?= libregression.a +PYTHON ?= python +ifeq ($(shell uname), Darwin) + DOWNLOAD?=curl -L -o +else + DOWNLOAD?=wget -O +endif +CORPORA_URL_PREFIX:=https://github.com/facebook/zstd/releases/download/fuzz-corpora/ ZSTDDIR = ../../lib PRGDIR = ../../programs FUZZ_CPPFLAGS := -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) \ - -DZSTD_DEBUG=1 -DMEM_FORCE_MEMORY_ACCESS=0 \ - -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION $(CPPFLAGS) -FUZZ_CFLAGS := -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ + $(CPPFLAGS) +FUZZ_EXTRA_FLAGS := -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ -Wstrict-prototypes -Wundef -Wformat-security \ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ -Wredundant-decls \ - -g -fno-omit-frame-pointer $(CFLAGS) -FUZZ_CXXFLAGS := -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ - -Wstrict-aliasing=1 -Wswitch-enum \ - -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \ - -Wformat-security -Wvla -Wformat=2 -Winit-self -Wfloat-equal \ - -Wwrite-strings -Wredundant-decls \ - -g -fno-omit-frame-pointer -std=c++11 $(CXXFLAGS) + -g -fno-omit-frame-pointer +FUZZ_CFLAGS := $(FUZZ_EXTRA_FLAGS) $(CFLAGS) +FUZZ_CXXFLAGS := $(FUZZ_EXTRA_FLAGS) -std=c++11 $(CXXFLAGS) FUZZ_LDFLAGS := $(LDFLAGS) FUZZ_ARFLAGS := $(ARFLAGS) FUZZ_TARGET_FLAGS = $(FUZZ_CPPFLAGS) $(FUZZ_CXXFLAGS) $(FUZZ_LDFLAGS) -FUZZ_HEADERS := fuzz_helpers.h fuzz.h +FUZZ_HEADERS := fuzz_helpers.h fuzz.h zstd_helpers.h +FUZZ_SRC := zstd_helpers.c -ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c -ZSTDCOMP_FILES := $(ZSTDDIR)/compress/*.c -ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c -ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) +ZSTDCOMMON_SRC := $(ZSTDDIR)/common/*.c +ZSTDCOMP_SRC := $(ZSTDDIR)/compress/*.c +ZSTDDECOMP_SRC := $(ZSTDDIR)/decompress/*.c +FUZZ_SRC := \ + $(FUZZ_SRC) \ + $(ZSTDDECOMP_SRC) \ + $(ZSTDCOMMON_SRC) \ + $(ZSTDCOMP_SRC) -ZSTD_OBJ := $(patsubst %.c,%.o, $(wildcard $(ZSTD_FILES))) +FUZZ_OBJ := $(patsubst %.c,%.o, $(wildcard $(FUZZ_SRC))) -LIBFUZZER ?= -lFuzzer -.PHONY: default all clean +.PHONY: default all clean cleanall default: all -all: round_trip simple_decompress +FUZZ_TARGETS := \ + simple_round_trip \ + stream_round_trip \ + block_round_trip \ + simple_decompress \ + stream_decompress \ + block_decompress + +all: $(FUZZ_TARGETS) %.o: %.c $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $^ -c -o $@ -simple_round_trip: $(FUZZ_HEADERS) $(ZSTD_OBJ) simple_round_trip.o - $(CXX) $(FUZZ_TARGET_FLAGS) $(ZSTD_OBJ) simple_round_trip.o $(LIBFUZZER) -o $@ - -stream_round_trip: $(FUZZ_HEADERS) $(ZSTD_OBJ) stream_round_trip.o - $(CXX) $(FUZZ_TARGET_FLAGS) $(ZSTD_OBJ) stream_round_trip.o $(LIBFUZZER) -o $@ - -simple_decompress: $(FUZZ_HEADERS) $(ZSTD_OBJ) simple_decompress.o - $(CXX) $(FUZZ_TARGET_FLAGS) $(ZSTD_OBJ) simple_decompress.o $(LIBFUZZER) -o $@ +simple_round_trip: $(FUZZ_HEADERS) $(FUZZ_OBJ) simple_round_trip.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) simple_round_trip.o $(LIB_FUZZING_ENGINE) -o $@ -stream_decompress: $(FUZZ_HEADERS) $(ZSTD_OBJ) stream_decompress.o - $(CXX) $(FUZZ_TARGET_FLAGS) $(ZSTD_OBJ) stream_decompress.o $(LIBFUZZER) -o $@ - -libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h regression_driver.o - $(AR) $(FUZZ_ARFLAGS) $@ regression_driver.o +stream_round_trip: $(FUZZ_HEADERS) $(FUZZ_OBJ) stream_round_trip.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) stream_round_trip.o $(LIB_FUZZING_ENGINE) -o $@ -%-regression: libregression.a - $(RM) $* - $(MAKE) $* LDFLAGS="$(FUZZ_LDFLAGS) -L." LIBFUZZER=-lregression +block_round_trip: $(FUZZ_HEADERS) $(FUZZ_OBJ) block_round_trip.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) block_round_trip.o $(LIB_FUZZING_ENGINE) -o $@ -%-regression-test: %-regression - ./$* corpora/$* +simple_decompress: $(FUZZ_HEADERS) $(FUZZ_OBJ) simple_decompress.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) simple_decompress.o $(LIB_FUZZING_ENGINE) -o $@ -regression-test: \ - simple_round_trip-regression-test \ - stream_round_trip-regression-test \ - simple_decompress-regression-test \ - stream_decompress-regression-test +stream_decompress: $(FUZZ_HEADERS) $(FUZZ_OBJ) stream_decompress.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) stream_decompress.o $(LIB_FUZZING_ENGINE) -o $@ -%-msan: clean - $(MAKE) $* CFLAGS="-fsanitize=memory $(FUZZ_CFLAGS)" \ - CXXFLAGS="-fsanitize=memory $(FUZZ_CXXFLAGS)" +block_decompress: $(FUZZ_HEADERS) $(FUZZ_OBJ) block_decompress.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) block_decompress.o $(LIB_FUZZING_ENGINE) -o $@ -UASAN_FLAGS := -fsanitize=address,undefined -fno-sanitize-recover=undefined \ - -fno-sanitize=pointer-overflow -%-uasan: clean - $(MAKE) $* CFLAGS="$(FUZZ_CFLAGS) $(UASAN_FLAGS)" \ - CXXFLAGS="$(FUZZ_CXXFLAGS) $(UASAN_FLAGS)" +libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h regression_driver.o + $(AR) $(FUZZ_ARFLAGS) $@ regression_driver.o # Install libfuzzer (not usable for MSAN testing) # Provided for convienence. To use this library run make libFuzzer and @@ -100,9 +100,28 @@ UASAN_FLAGS := -fsanitize=address,undefined -fno-sanitize-recover=undefined \ libFuzzer: @$(RM) -rf Fuzzer @git clone https://chromium.googlesource.com/chromium/llvm-project/llvm/lib/Fuzzer - @./Fuzzer/build.sh + @cd Fuzzer && ./build.sh + +corpora/%_seed_corpus.zip: + @mkdir -p corpora + $(DOWNLOAD) $@ $(CORPORA_URL_PREFIX)$*_seed_corpus.zip + +corpora/%: corpora/%_seed_corpus.zip + unzip -q $^ -d $@ + +.PHONY: corpora +corpora: $(patsubst %,corpora/%,$(FUZZ_TARGETS)) + +regressiontest: corpora + CC="$(CC)" CXX="$(CXX)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" LDFLAGS="$(LDFLAGS)" $(PYTHON) ./fuzz.py build all + $(PYTHON) ./fuzz.py regression all clean: @$(MAKE) -C $(ZSTDDIR) clean - @$(RM) -f *.a *.o - @$(RM) -f simple_round_trip stream_round_trip simple_decompress stream_decompress + @$(RM) *.a *.o + @$(RM) simple_round_trip stream_round_trip simple_decompress \ + stream_decompress block_decompress block_round_trip + +cleanall: + @$(RM) -r Fuzzer + @$(RM) -r corpora diff --git a/tests/fuzz/README.md b/tests/fuzz/README.md index 38a4f3d1ab02..f184be646ef6 100644 --- a/tests/fuzz/README.md +++ b/tests/fuzz/README.md @@ -1,34 +1,96 @@ # Fuzzing Each fuzzing target can be built with multiple engines. +Zstd provides a fuzz corpus for each target that can be downloaded with +the command: + +``` +make corpora +``` + +It will download each corpus into `./corpora/TARGET`. + +## fuzz.py + +`fuzz.py` is a helper script for building and running fuzzers. +Run `./fuzz.py -h` for the commands and run `./fuzz.py COMMAND -h` for +command specific help. + +### Generating Data + +`fuzz.py` provides a utility to generate seed data for each fuzzer. + +``` +make -C ../tests decodecorpus +./fuzz.py gen TARGET +``` + +By default it outputs 100 samples, each at most 8KB into `corpora/TARGET-seed`, +but that can be configured with the `--number`, `--max-size-log` and `--seed` +flags. + +### Build +It respects the usual build environment variables `CC`, `CFLAGS`, etc. +The environment variables can be overridden with the corresponding flags +`--cc`, `--cflags`, etc. +The specific fuzzing engine is selected with `LIB_FUZZING_ENGINE` or +`--lib-fuzzing-engine`, the default is `libregression.a`. +It has flags that can easily set up sanitizers `--enable-{a,ub,m}san`, and +coverage instrumentation `--enable-coverage`. +It sets sane defaults which can be overriden with flags `--debug`, +`--enable-ubsan-pointer-overlow`, etc. +Run `./fuzz.py build -h` for help. + +### Running Fuzzers + +`./fuzz.py` can run `libfuzzer`, `afl`, and `regression` tests. +See the help of the relevant command for options. +Flags not parsed by `fuzz.py` are passed to the fuzzing engine. +The command used to run the fuzzer is printed for debugging. ## LibFuzzer -You can install `libFuzzer` with `make libFuzzer`. Then you can make each target -with `make target LDFLAGS=-L. CC=clang CXX=clang++`. +``` +# Build libfuzzer if necessary +make libFuzzer +# Build the fuzz targets +./fuzz.py build all --enable-coverage --enable-asan --enable-ubsan --lib-fuzzing-engine Fuzzer/libFuzzer.a --cc clang --cxx clang++ +# OR equivalently +CC=clang CXX=clang++ LIB_FUZZING_ENGINE=Fuzzer/libFuzzer.a ./fuzz.py build all --enable-coverage --enable-asan --enable-ubsan +# Run the fuzzer +./fuzz.py libfuzzer TARGET -max_len=8192 -jobs=4 +``` + +where `TARGET` could be `simple_decompress`, `stream_round_trip`, etc. + +### MSAN + +Fuzzing with `libFuzzer` and `MSAN` will require building a C++ standard library +and libFuzzer with MSAN. +`fuzz.py` respects the environment variables / flags `MSAN_EXTRA_CPPFLAGS`, +`MSAN_EXTRA_CFLAGS`, `MSAN_EXTRA_CXXFLAGS`, `MSAN_EXTRA_LDFLAGS` to easily pass +the extra parameters only for MSAN. ## AFL -The regression driver also serves as a binary for `afl-fuzz`. You can make each -target with one of these commands: +The default `LIB_FUZZING_ENGINE` is `libregression.a`, which produces a binary +that AFL can use. ``` -make target-regression CC=afl-clang CXX=afl-clang++ -AFL_MSAN=1 make target-regression-msan CC=afl-clang CXX=afl-clang++ -AFL_ASAN=1 make target-regression-uasan CC=afl-clang CXX=afl-clang++ +# Build the fuzz targets +CC=afl-clang CXX=afl-clang++ ./fuzz.py build all --enable-asan --enable-ubsan +# Run the fuzzer without a memory limit because of ASAN +./fuzz.py afl TARGET -m none ``` -Then run as `./target @@`. - ## Regression Testing -Each fuzz target has a corpus checked into the repo under `fuzz/corpora/`. -You can run regression tests on the corpora to ensure that inputs which -previously exposed bugs still pass. You can make these targets to run the -regression tests with different sanitizers. +The regression rest supports the `all` target to run all the fuzzers in one +command. ``` -make regression-test -make regression-test-msan -make regression-test-uasan +CC=clang CXX=clang++ ./fuzz.py build all --enable-asan --enable-ubsan +./fuzz.py regression all +CC=clang CXX=clang++ ./fuzz.py build all --enable-msan +./fuzz.py regression all ``` diff --git a/tests/fuzz/block_decompress.c b/tests/fuzz/block_decompress.c new file mode 100644 index 000000000000..3cccc32f4e52 --- /dev/null +++ b/tests/fuzz/block_decompress.c @@ -0,0 +1,51 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + */ + +/** + * This fuzz target attempts to decompress the fuzzed data with the simple + * decompression function to ensure the decompressor never crashes. + */ + +#define ZSTD_STATIC_LINKING_ONLY + +#include <stddef.h> +#include <stdlib.h> +#include <stdio.h> +#include "fuzz_helpers.h" +#include "zstd.h" + +static ZSTD_DCtx *dctx = NULL; +static void* rBuf = NULL; +static size_t bufSize = 0; + +int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) +{ + size_t const neededBufSize = ZSTD_BLOCKSIZE_MAX; + + FUZZ_seed(&src, &size); + + /* Allocate all buffers and contexts if not already allocated */ + if (neededBufSize > bufSize) { + free(rBuf); + rBuf = malloc(neededBufSize); + bufSize = neededBufSize; + FUZZ_ASSERT(rBuf); + } + if (!dctx) { + dctx = ZSTD_createDCtx(); + FUZZ_ASSERT(dctx); + } + ZSTD_decompressBegin(dctx); + ZSTD_decompressBlock(dctx, rBuf, neededBufSize, src, size); + +#ifndef STATEFUL_FUZZING + ZSTD_freeDCtx(dctx); dctx = NULL; +#endif + return 0; +} diff --git a/tests/fuzz/block_round_trip.c b/tests/fuzz/block_round_trip.c new file mode 100644 index 000000000000..64ca5fc40f9c --- /dev/null +++ b/tests/fuzz/block_round_trip.c @@ -0,0 +1,92 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + */ + +/** + * This fuzz target performs a zstd round-trip test (compress & decompress), + * compares the result with the original, and calls abort() on corruption. + */ + +#define ZSTD_STATIC_LINKING_ONLY + +#include <stddef.h> +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include "fuzz_helpers.h" +#include "zstd.h" + +static const int kMaxClevel = 19; + +static ZSTD_CCtx *cctx = NULL; +static ZSTD_DCtx *dctx = NULL; +static void* cBuf = NULL; +static void* rBuf = NULL; +static size_t bufSize = 0; +static uint32_t seed; + +static size_t roundTripTest(void *result, size_t resultCapacity, + void *compressed, size_t compressedCapacity, + const void *src, size_t srcSize) +{ + int const cLevel = FUZZ_rand(&seed) % kMaxClevel; + ZSTD_parameters const params = ZSTD_getParams(cLevel, srcSize, 0); + size_t ret = ZSTD_compressBegin_advanced(cctx, NULL, 0, params, srcSize); + FUZZ_ZASSERT(ret); + + ret = ZSTD_compressBlock(cctx, compressed, compressedCapacity, src, srcSize); + FUZZ_ZASSERT(ret); + if (ret == 0) { + FUZZ_ASSERT(resultCapacity >= srcSize); + memcpy(result, src, srcSize); + return srcSize; + } + ZSTD_decompressBegin(dctx); + return ZSTD_decompressBlock(dctx, result, resultCapacity, compressed, ret); +} + +int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) +{ + size_t neededBufSize; + + seed = FUZZ_seed(&src, &size); + neededBufSize = size; + if (size > ZSTD_BLOCKSIZE_MAX) + return 0; + + /* Allocate all buffers and contexts if not already allocated */ + if (neededBufSize > bufSize || !cBuf || !rBuf) { + free(cBuf); + free(rBuf); + cBuf = malloc(neededBufSize); + rBuf = malloc(neededBufSize); + bufSize = neededBufSize; + FUZZ_ASSERT(cBuf && rBuf); + } + if (!cctx) { + cctx = ZSTD_createCCtx(); + FUZZ_ASSERT(cctx); + } + if (!dctx) { + dctx = ZSTD_createDCtx(); + FUZZ_ASSERT(dctx); + } + + { + size_t const result = + roundTripTest(rBuf, neededBufSize, cBuf, neededBufSize, src, size); + FUZZ_ZASSERT(result); + FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size"); + FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!"); + } +#ifndef STATEFUL_FUZZING + ZSTD_freeCCtx(cctx); cctx = NULL; + ZSTD_freeDCtx(dctx); dctx = NULL; +#endif + return 0; +} diff --git a/tests/fuzz/default.options b/tests/fuzz/default.options new file mode 100644 index 000000000000..8ea8588375d7 --- /dev/null +++ b/tests/fuzz/default.options @@ -0,0 +1,2 @@ +[libfuzzer] +max_len = 8192 diff --git a/tests/fuzz/fuzz.h b/tests/fuzz/fuzz.h index 5b71aba89b43..a64845473c2b 100644 --- a/tests/fuzz/fuzz.h +++ b/tests/fuzz/fuzz.h @@ -1,10 +1,10 @@ -/** +/* * Copyright (c) 2016-present, Facebook, Inc. * All rights reserved. * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). */ /** @@ -12,15 +12,17 @@ * Fuzz targets have some common parameters passed as macros during compilation. * Check the documentation for each individual fuzzer for more parameters. * - * @param STATEFULL_FUZZING: + * @param STATEFUL_FUZZING: * Define this to reuse state between fuzzer runs. This can be useful to * test code paths which are only executed when contexts are reused. * WARNING: Makes reproducing crashes much harder. * Default: Not defined. * @param FUZZ_RNG_SEED_SIZE: * The number of bytes of the source to look at when constructing a seed - * for the deterministic RNG. - * Default: 128. + * for the deterministic RNG. These bytes are discarded before passing + * the data to zstd functions. Every fuzzer initializes the RNG exactly + * once before doing anything else, even if it is unused. + * Default: 4. * @param ZSTD_DEBUG: * This is a parameter for the zstd library. Defining `ZSTD_DEBUG=1` * enables assert() statements in the zstd library. Higher levels enable @@ -41,12 +43,20 @@ #define FUZZ_H #ifndef FUZZ_RNG_SEED_SIZE -# define FUZZ_RNG_SEED_SIZE 128 +# define FUZZ_RNG_SEED_SIZE 4 #endif #include <stddef.h> #include <stdint.h> +#ifdef __cplusplus +extern "C" { +#endif + int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size); +#ifdef __cplusplus +} +#endif + #endif diff --git a/tests/fuzz/fuzz.py b/tests/fuzz/fuzz.py new file mode 100755 index 000000000000..b591e4f6734e --- /dev/null +++ b/tests/fuzz/fuzz.py @@ -0,0 +1,818 @@ +#!/usr/bin/env python + +# ################################################################ +# Copyright (c) 2016-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ########################################################################## + +import argparse +import contextlib +import os +import re +import shutil +import subprocess +import sys +import tempfile + + +def abs_join(a, *p): + return os.path.abspath(os.path.join(a, *p)) + + +# Constants +FUZZ_DIR = os.path.abspath(os.path.dirname(__file__)) +CORPORA_DIR = abs_join(FUZZ_DIR, 'corpora') +TARGETS = [ + 'simple_round_trip', + 'stream_round_trip', + 'block_round_trip', + 'simple_decompress', + 'stream_decompress', + 'block_decompress', +] +ALL_TARGETS = TARGETS + ['all'] +FUZZ_RNG_SEED_SIZE = 4 + +# Standard environment variables +CC = os.environ.get('CC', 'cc') +CXX = os.environ.get('CXX', 'c++') +CPPFLAGS = os.environ.get('CPPFLAGS', '') +CFLAGS = os.environ.get('CFLAGS', '-O3') +CXXFLAGS = os.environ.get('CXXFLAGS', CFLAGS) +LDFLAGS = os.environ.get('LDFLAGS', '') +MFLAGS = os.environ.get('MFLAGS', '-j') + +# Fuzzing environment variables +LIB_FUZZING_ENGINE = os.environ.get('LIB_FUZZING_ENGINE', 'libregression.a') +AFL_FUZZ = os.environ.get('AFL_FUZZ', 'afl-fuzz') +DECODECORPUS = os.environ.get('DECODECORPUS', + abs_join(FUZZ_DIR, '..', 'decodecorpus')) + +# Sanitizer environment variables +MSAN_EXTRA_CPPFLAGS = os.environ.get('MSAN_EXTRA_CPPFLAGS', '') +MSAN_EXTRA_CFLAGS = os.environ.get('MSAN_EXTRA_CFLAGS', '') +MSAN_EXTRA_CXXFLAGS = os.environ.get('MSAN_EXTRA_CXXFLAGS', '') +MSAN_EXTRA_LDFLAGS = os.environ.get('MSAN_EXTRA_LDFLAGS', '') + + +def create(r): + d = os.path.abspath(r) + if not os.path.isdir(d): + os.mkdir(d) + return d + + +def check(r): + d = os.path.abspath(r) + if not os.path.isdir(d): + return None + return d + + +@contextlib.contextmanager +def tmpdir(): + dirpath = tempfile.mkdtemp() + try: + yield dirpath + finally: + shutil.rmtree(dirpath, ignore_errors=True) + + +def parse_targets(in_targets): + targets = set() + for target in in_targets: + if not target: + continue + if target == 'all': + targets = targets.union(TARGETS) + elif target in TARGETS: + targets.add(target) + else: + raise RuntimeError('{} is not a valid target'.format(target)) + return list(targets) + + +def targets_parser(args, description): + parser = argparse.ArgumentParser(prog=args.pop(0), description=description) + parser.add_argument( + 'TARGET', + nargs='*', + type=str, + help='Fuzz target(s) to build {{{}}}'.format(', '.join(ALL_TARGETS))) + args, extra = parser.parse_known_args(args) + args.extra = extra + + args.TARGET = parse_targets(args.TARGET) + + return args + + +def parse_env_flags(args, flags): + """ + Look for flags set by environment variables. + """ + san_flags = ','.join(re.findall('-fsanitize=((?:[a-z]+,?)+)', flags)) + nosan_flags = ','.join(re.findall('-fno-sanitize=((?:[a-z]+,?)+)', flags)) + + def set_sanitizer(sanitizer, default, san, nosan): + if sanitizer in san and sanitizer in nosan: + raise RuntimeError('-fno-sanitize={s} and -fsanitize={s} passed'. + format(s=sanitizer)) + if sanitizer in san: + return True + if sanitizer in nosan: + return False + return default + + san = set(san_flags.split(',')) + nosan = set(nosan_flags.split(',')) + + args.asan = set_sanitizer('address', args.asan, san, nosan) + args.msan = set_sanitizer('memory', args.msan, san, nosan) + args.ubsan = set_sanitizer('undefined', args.ubsan, san, nosan) + + args.sanitize = args.asan or args.msan or args.ubsan + + return args + + +def compiler_version(cc, cxx): + """ + Determines the compiler and version. + Only works for clang and gcc. + """ + cc_version_bytes = subprocess.check_output([cc, "--version"]) + cxx_version_bytes = subprocess.check_output([cxx, "--version"]) + if cc_version_bytes.startswith(b'clang'): + assert(cxx_version_bytes.startswith(b'clang')) + compiler = 'clang' + if cc_version_bytes.startswith(b'gcc'): + assert(cxx_version_bytes.startswith(b'g++')) + compiler = 'gcc' + version_regex = b'([0-9])+\.([0-9])+\.([0-9])+' + version_match = re.search(version_regex, cc_version_bytes) + version = tuple(int(version_match.group(i)) for i in range(1, 4)) + return compiler, version + + +def overflow_ubsan_flags(cc, cxx): + compiler, version = compiler_version(cc, cxx) + if compiler == 'gcc': + return ['-fno-sanitize=signed-integer-overflow'] + if compiler == 'clang' and version >= (5, 0, 0): + return ['-fno-sanitize=pointer-overflow'] + return [] + + +def build_parser(args): + description = """ + Cleans the repository and builds a fuzz target (or all). + Many flags default to environment variables (default says $X='y'). + Options that aren't enabling features default to the correct values for + zstd. + Enable sanitizers with --enable-*san. + For regression testing just build. + For libFuzzer set LIB_FUZZING_ENGINE and pass --enable-coverage. + For AFL set CC and CXX to AFL's compilers and set + LIB_FUZZING_ENGINE='libregression.a'. + """ + parser = argparse.ArgumentParser(prog=args.pop(0), description=description) + parser.add_argument( + '--lib-fuzzing-engine', + dest='lib_fuzzing_engine', + type=str, + default=LIB_FUZZING_ENGINE, + help=('The fuzzing engine to use e.g. /path/to/libFuzzer.a ' + "(default: $LIB_FUZZING_ENGINE='{})".format(LIB_FUZZING_ENGINE))) + parser.add_argument( + '--enable-coverage', + dest='coverage', + action='store_true', + help='Enable coverage instrumentation (-fsanitize-coverage)') + parser.add_argument( + '--enable-asan', dest='asan', action='store_true', help='Enable UBSAN') + parser.add_argument( + '--enable-ubsan', + dest='ubsan', + action='store_true', + help='Enable UBSAN') + parser.add_argument( + '--enable-ubsan-pointer-overflow', + dest='ubsan_pointer_overflow', + action='store_true', + help='Enable UBSAN pointer overflow check (known failure)') + parser.add_argument( + '--enable-msan', dest='msan', action='store_true', help='Enable MSAN') + parser.add_argument( + '--enable-msan-track-origins', dest='msan_track_origins', + action='store_true', help='Enable MSAN origin tracking') + parser.add_argument( + '--msan-extra-cppflags', + dest='msan_extra_cppflags', + type=str, + default=MSAN_EXTRA_CPPFLAGS, + help="Extra CPPFLAGS for MSAN (default: $MSAN_EXTRA_CPPFLAGS='{}')". + format(MSAN_EXTRA_CPPFLAGS)) + parser.add_argument( + '--msan-extra-cflags', + dest='msan_extra_cflags', + type=str, + default=MSAN_EXTRA_CFLAGS, + help="Extra CFLAGS for MSAN (default: $MSAN_EXTRA_CFLAGS='{}')".format( + MSAN_EXTRA_CFLAGS)) + parser.add_argument( + '--msan-extra-cxxflags', + dest='msan_extra_cxxflags', + type=str, + default=MSAN_EXTRA_CXXFLAGS, + help="Extra CXXFLAGS for MSAN (default: $MSAN_EXTRA_CXXFLAGS='{}')". + format(MSAN_EXTRA_CXXFLAGS)) + parser.add_argument( + '--msan-extra-ldflags', + dest='msan_extra_ldflags', + type=str, + default=MSAN_EXTRA_LDFLAGS, + help="Extra LDFLAGS for MSAN (default: $MSAN_EXTRA_LDFLAGS='{}')". + format(MSAN_EXTRA_LDFLAGS)) + parser.add_argument( + '--enable-sanitize-recover', + dest='sanitize_recover', + action='store_true', + help='Non-fatal sanitizer errors where possible') + parser.add_argument( + '--debug', + dest='debug', + type=int, + default=1, + help='Set ZSTD_DEBUG (default: 1)') + parser.add_argument( + '--force-memory-access', + dest='memory_access', + type=int, + default=0, + help='Set MEM_FORCE_MEMORY_ACCESS (default: 0)') + parser.add_argument( + '--fuzz-rng-seed-size', + dest='fuzz_rng_seed_size', + type=int, + default=4, + help='Set FUZZ_RNG_SEED_SIZE (default: 4)') + parser.add_argument( + '--disable-fuzzing-mode', + dest='fuzzing_mode', + action='store_false', + help='Do not define FUZZING_BUILD_MORE_UNSAFE_FOR_PRODUCTION') + parser.add_argument( + '--enable-stateful-fuzzing', + dest='stateful_fuzzing', + action='store_true', + help='Reuse contexts between runs (makes reproduction impossible)') + parser.add_argument( + '--cc', + dest='cc', + type=str, + default=CC, + help="CC (default: $CC='{}')".format(CC)) + parser.add_argument( + '--cxx', + dest='cxx', + type=str, + default=CXX, + help="CXX (default: $CXX='{}')".format(CXX)) + parser.add_argument( + '--cppflags', + dest='cppflags', + type=str, + default=CPPFLAGS, + help="CPPFLAGS (default: $CPPFLAGS='{}')".format(CPPFLAGS)) + parser.add_argument( + '--cflags', + dest='cflags', + type=str, + default=CFLAGS, + help="CFLAGS (default: $CFLAGS='{}')".format(CFLAGS)) + parser.add_argument( + '--cxxflags', + dest='cxxflags', + type=str, + default=CXXFLAGS, + help="CXXFLAGS (default: $CXXFLAGS='{}')".format(CXXFLAGS)) + parser.add_argument( + '--ldflags', + dest='ldflags', + type=str, + default=LDFLAGS, + help="LDFLAGS (default: $LDFLAGS='{}')".format(LDFLAGS)) + parser.add_argument( + '--mflags', + dest='mflags', + type=str, + default=MFLAGS, + help="Extra Make flags (default: $MFLAGS='{}')".format(MFLAGS)) + parser.add_argument( + 'TARGET', + nargs='*', + type=str, + help='Fuzz target(s) to build {{{}}}'.format(', '.join(ALL_TARGETS)) + ) + args = parser.parse_args(args) + args = parse_env_flags(args, ' '.join( + [args.cppflags, args.cflags, args.cxxflags, args.ldflags])) + + # Check option sanitiy + if args.msan and (args.asan or args.ubsan): + raise RuntimeError('MSAN may not be used with any other sanitizers') + if args.msan_track_origins and not args.msan: + raise RuntimeError('--enable-msan-track-origins requires MSAN') + if args.ubsan_pointer_overflow and not args.ubsan: + raise RuntimeError('--enable-ubsan-pointer-overlow requires UBSAN') + if args.sanitize_recover and not args.sanitize: + raise RuntimeError('--enable-sanitize-recover but no sanitizers used') + + return args + + +def build(args): + try: + args = build_parser(args) + except Exception as e: + print(e) + return 1 + # The compilation flags we are setting + targets = args.TARGET + cc = args.cc + cxx = args.cxx + cppflags = [args.cppflags] + cflags = [args.cflags] + ldflags = [args.ldflags] + cxxflags = [args.cxxflags] + mflags = [args.mflags] if args.mflags else [] + # Flags to be added to both cflags and cxxflags + common_flags = [] + + cppflags += [ + '-DZSTD_DEBUG={}'.format(args.debug), + '-DMEM_FORCE_MEMORY_ACCESS={}'.format(args.memory_access), + '-DFUZZ_RNG_SEED_SIZE={}'.format(args.fuzz_rng_seed_size), + ] + + mflags += ['LIB_FUZZING_ENGINE={}'.format(args.lib_fuzzing_engine)] + + # Set flags for options + if args.coverage: + common_flags += [ + '-fsanitize-coverage=trace-pc-guard,indirect-calls,trace-cmp' + ] + + if args.sanitize_recover: + recover_flags = ['-fsanitize-recover=all'] + else: + recover_flags = ['-fno-sanitize-recover=all'] + if args.sanitize: + common_flags += recover_flags + + if args.msan: + msan_flags = ['-fsanitize=memory'] + if args.msan_track_origins: + msan_flags += ['-fsanitize-memory-track-origins'] + common_flags += msan_flags + # Append extra MSAN flags (it might require special setup) + cppflags += [args.msan_extra_cppflags] + cflags += [args.msan_extra_cflags] + cxxflags += [args.msan_extra_cxxflags] + ldflags += [args.msan_extra_ldflags] + + if args.asan: + common_flags += ['-fsanitize=address'] + + if args.ubsan: + ubsan_flags = ['-fsanitize=undefined'] + if not args.ubsan_pointer_overflow: + ubsan_flags += overflow_ubsan_flags(cc, cxx) + common_flags += ubsan_flags + + if args.stateful_fuzzing: + cppflags += ['-DSTATEFUL_FUZZING'] + + if args.fuzzing_mode: + cppflags += ['-DFUZZING_BUILD_MORE_UNSAFE_FOR_PRODUCTION'] + + if args.lib_fuzzing_engine == 'libregression.a': + targets = ['libregression.a'] + targets + + # Append the common flags + cflags += common_flags + cxxflags += common_flags + + # Prepare the flags for Make + cc_str = "CC={}".format(cc) + cxx_str = "CXX={}".format(cxx) + cppflags_str = "CPPFLAGS={}".format(' '.join(cppflags)) + cflags_str = "CFLAGS={}".format(' '.join(cflags)) + cxxflags_str = "CXXFLAGS={}".format(' '.join(cxxflags)) + ldflags_str = "LDFLAGS={}".format(' '.join(ldflags)) + + # Print the flags + print('MFLAGS={}'.format(' '.join(mflags))) + print(cc_str) + print(cxx_str) + print(cppflags_str) + print(cflags_str) + print(cxxflags_str) + print(ldflags_str) + + # Clean and build + clean_cmd = ['make', 'clean'] + mflags + print(' '.join(clean_cmd)) + subprocess.check_call(clean_cmd) + build_cmd = [ + 'make', + cc_str, + cxx_str, + cppflags_str, + cflags_str, + cxxflags_str, + ldflags_str, + ] + mflags + targets + print(' '.join(build_cmd)) + subprocess.check_call(build_cmd) + return 0 + + +def libfuzzer_parser(args): + description = """ + Runs a libfuzzer binary. + Passes all extra arguments to libfuzzer. + The fuzzer should have been build with LIB_FUZZING_ENGINE pointing to + libFuzzer.a. + Generates output in the CORPORA directory, puts crashes in the ARTIFACT + directory, and takes extra input from the SEED directory. + To merge AFL's output pass the SEED as AFL's output directory and pass + '-merge=1'. + """ + parser = argparse.ArgumentParser(prog=args.pop(0), description=description) + parser.add_argument( + '--corpora', + type=str, + help='Override the default corpora dir (default: {})'.format( + abs_join(CORPORA_DIR, 'TARGET'))) + parser.add_argument( + '--artifact', + type=str, + help='Override the default artifact dir (default: {})'.format( + abs_join(CORPORA_DIR, 'TARGET-crash'))) + parser.add_argument( + '--seed', + type=str, + help='Override the default seed dir (default: {})'.format( + abs_join(CORPORA_DIR, 'TARGET-seed'))) + parser.add_argument( + 'TARGET', + type=str, + help='Fuzz target(s) to build {{{}}}'.format(', '.join(TARGETS))) + args, extra = parser.parse_known_args(args) + args.extra = extra + + if args.TARGET and args.TARGET not in TARGETS: + raise RuntimeError('{} is not a valid target'.format(args.TARGET)) + + return args + + +def libfuzzer(target, corpora=None, artifact=None, seed=None, extra_args=None): + if corpora is None: + corpora = abs_join(CORPORA_DIR, target) + if artifact is None: + artifact = abs_join(CORPORA_DIR, '{}-crash'.format(target)) + if seed is None: + seed = abs_join(CORPORA_DIR, '{}-seed'.format(target)) + if extra_args is None: + extra_args = [] + + target = abs_join(FUZZ_DIR, target) + + corpora = [create(corpora)] + artifact = create(artifact) + seed = check(seed) + + corpora += [artifact] + if seed is not None: + corpora += [seed] + + cmd = [target, '-artifact_prefix={}/'.format(artifact)] + cmd += corpora + extra_args + print(' '.join(cmd)) + subprocess.check_call(cmd) + + +def libfuzzer_cmd(args): + try: + args = libfuzzer_parser(args) + except Exception as e: + print(e) + return 1 + libfuzzer(args.TARGET, args.corpora, args.artifact, args.seed, args.extra) + return 0 + + +def afl_parser(args): + description = """ + Runs an afl-fuzz job. + Passes all extra arguments to afl-fuzz. + The fuzzer should have been built with CC/CXX set to the AFL compilers, + and with LIB_FUZZING_ENGINE='libregression.a'. + Takes input from CORPORA and writes output to OUTPUT. + Uses AFL_FUZZ as the binary (set from flag or environment variable). + """ + parser = argparse.ArgumentParser(prog=args.pop(0), description=description) + parser.add_argument( + '--corpora', + type=str, + help='Override the default corpora dir (default: {})'.format( + abs_join(CORPORA_DIR, 'TARGET'))) + parser.add_argument( + '--output', + type=str, + help='Override the default AFL output dir (default: {})'.format( + abs_join(CORPORA_DIR, 'TARGET-afl'))) + parser.add_argument( + '--afl-fuzz', + type=str, + default=AFL_FUZZ, + help='AFL_FUZZ (default: $AFL_FUZZ={})'.format(AFL_FUZZ)) + parser.add_argument( + 'TARGET', + type=str, + help='Fuzz target(s) to build {{{}}}'.format(', '.join(TARGETS))) + args, extra = parser.parse_known_args(args) + args.extra = extra + + if args.TARGET and args.TARGET not in TARGETS: + raise RuntimeError('{} is not a valid target'.format(args.TARGET)) + + if not args.corpora: + args.corpora = abs_join(CORPORA_DIR, args.TARGET) + if not args.output: + args.output = abs_join(CORPORA_DIR, '{}-afl'.format(args.TARGET)) + + return args + + +def afl(args): + try: + args = afl_parser(args) + except Exception as e: + print(e) + return 1 + target = abs_join(FUZZ_DIR, args.TARGET) + + corpora = create(args.corpora) + output = create(args.output) + + cmd = [args.afl_fuzz, '-i', corpora, '-o', output] + args.extra + cmd += [target, '@@'] + print(' '.join(cmd)) + subprocess.call(cmd) + return 0 + + +def regression(args): + try: + description = """ + Runs one or more regression tests. + The fuzzer should have been built with with + LIB_FUZZING_ENGINE='libregression.a'. + Takes input from CORPORA. + """ + args = targets_parser(args, description) + except Exception as e: + print(e) + return 1 + for target in args.TARGET: + corpora = create(abs_join(CORPORA_DIR, target)) + target = abs_join(FUZZ_DIR, target) + cmd = [target, corpora] + print(' '.join(cmd)) + subprocess.check_call(cmd) + return 0 + + +def gen_parser(args): + description = """ + Generate a seed corpus appropiate for TARGET with data generated with + decodecorpus. + The fuzz inputs are prepended with a seed before the zstd data, so the + output of decodecorpus shouldn't be used directly. + Generates NUMBER samples prepended with FUZZ_RNG_SEED_SIZE random bytes and + puts the output in SEED. + DECODECORPUS is the decodecorpus binary, and must already be built. + """ + parser = argparse.ArgumentParser(prog=args.pop(0), description=description) + parser.add_argument( + '--number', + '-n', + type=int, + default=100, + help='Number of samples to generate') + parser.add_argument( + '--max-size-log', + type=int, + default=13, + help='Maximum sample size to generate') + parser.add_argument( + '--seed', + type=str, + help='Override the default seed dir (default: {})'.format( + abs_join(CORPORA_DIR, 'TARGET-seed'))) + parser.add_argument( + '--decodecorpus', + type=str, + default=DECODECORPUS, + help="decodecorpus binary (default: $DECODECORPUS='{}')".format( + DECODECORPUS)) + parser.add_argument( + '--fuzz-rng-seed-size', + type=int, + default=4, + help="FUZZ_RNG_SEED_SIZE used for generate the samples (must match)" + ) + parser.add_argument( + 'TARGET', + type=str, + help='Fuzz target(s) to build {{{}}}'.format(', '.join(TARGETS))) + args, extra = parser.parse_known_args(args) + args.extra = extra + + if args.TARGET and args.TARGET not in TARGETS: + raise RuntimeError('{} is not a valid target'.format(args.TARGET)) + + if not args.seed: + args.seed = abs_join(CORPORA_DIR, '{}-seed'.format(args.TARGET)) + + if not os.path.isfile(args.decodecorpus): + raise RuntimeError("{} is not a file run 'make -C {} decodecorpus'". + format(args.decodecorpus, abs_join(FUZZ_DIR, '..'))) + + return args + + +def gen(args): + try: + args = gen_parser(args) + except Exception as e: + print(e) + return 1 + + seed = create(args.seed) + with tmpdir() as compressed: + with tmpdir() as decompressed: + cmd = [ + args.decodecorpus, + '-n{}'.format(args.number), + '-p{}/'.format(compressed), + '-o{}'.format(decompressed), + ] + + if 'block_' in args.TARGET: + cmd += [ + '--gen-blocks', + '--max-block-size-log={}'.format(args.max_size_log) + ] + else: + cmd += ['--max-content-size-log={}'.format(args.max_size_log)] + + print(' '.join(cmd)) + subprocess.check_call(cmd) + + if '_round_trip' in args.TARGET: + print('using decompressed data in {}'.format(decompressed)) + samples = decompressed + elif '_decompress' in args.TARGET: + print('using compressed data in {}'.format(compressed)) + samples = compressed + + # Copy the samples over and prepend the RNG seeds + for name in os.listdir(samples): + samplename = abs_join(samples, name) + outname = abs_join(seed, name) + rng_seed = os.urandom(args.fuzz_rng_seed_size) + with open(samplename, 'rb') as sample: + with open(outname, 'wb') as out: + out.write(rng_seed) + CHUNK_SIZE = 131072 + chunk = sample.read(CHUNK_SIZE) + while len(chunk) > 0: + out.write(chunk) + chunk = sample.read(CHUNK_SIZE) + return 0 + + +def minimize(args): + try: + description = """ + Runs a libfuzzer fuzzer with -merge=1 to build a minimal corpus in + TARGET_seed_corpus. All extra args are passed to libfuzzer. + """ + args = targets_parser(args, description) + except Exception as e: + print(e) + return 1 + + for target in args.TARGET: + # Merge the corpus + anything else into the seed_corpus + corpus = abs_join(CORPORA_DIR, target) + seed_corpus = abs_join(CORPORA_DIR, "{}_seed_corpus".format(target)) + extra_args = [corpus, "-merge=1"] + args.extra + libfuzzer(target, corpora=seed_corpus, extra_args=extra_args) + seeds = set(os.listdir(seed_corpus)) + # Copy all crashes directly into the seed_corpus if not already present + crashes = abs_join(CORPORA_DIR, '{}-crash'.format(target)) + for crash in os.listdir(crashes): + if crash not in seeds: + shutil.copy(abs_join(crashes, crash), seed_corpus) + seeds.add(crash) + + +def zip_cmd(args): + try: + description = """ + Zips up the seed corpus. + """ + args = targets_parser(args, description) + except Exception as e: + print(e) + return 1 + + for target in args.TARGET: + # Zip the seed_corpus + seed_corpus = abs_join(CORPORA_DIR, "{}_seed_corpus".format(target)) + seeds = [abs_join(seed_corpus, f) for f in os.listdir(seed_corpus)] + zip_file = "{}.zip".format(seed_corpus) + cmd = ["zip", "-q", "-j", "-9", zip_file] + print(' '.join(cmd + [abs_join(seed_corpus, '*')])) + subprocess.check_call(cmd + seeds) + + +def list_cmd(args): + print("\n".join(TARGETS)) + + +def short_help(args): + name = args[0] + print("Usage: {} [OPTIONS] COMMAND [ARGS]...\n".format(name)) + + +def help(args): + short_help(args) + print("\tfuzzing helpers (select a command and pass -h for help)\n") + print("Options:") + print("\t-h, --help\tPrint this message") + print("") + print("Commands:") + print("\tbuild\t\tBuild a fuzzer") + print("\tlibfuzzer\tRun a libFuzzer fuzzer") + print("\tafl\t\tRun an AFL fuzzer") + print("\tregression\tRun a regression test") + print("\tgen\t\tGenerate a seed corpus for a fuzzer") + print("\tminimize\tMinimize the test corpora") + print("\tzip\t\tZip the minimized corpora up") + print("\tlist\t\tList the available targets") + + +def main(): + args = sys.argv + if len(args) < 2: + help(args) + return 1 + if args[1] == '-h' or args[1] == '--help' or args[1] == '-H': + help(args) + return 1 + command = args.pop(1) + args[0] = "{} {}".format(args[0], command) + if command == "build": + return build(args) + if command == "libfuzzer": + return libfuzzer_cmd(args) + if command == "regression": + return regression(args) + if command == "afl": + return afl(args) + if command == "gen": + return gen(args) + if command == "minimize": + return minimize(args) + if command == "zip": + return zip_cmd(args) + if command == "list": + return list_cmd(args) + short_help(args) + print("Error: No such command {} (pass -h for help)".format(command)) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/fuzz/fuzz_helpers.h b/tests/fuzz/fuzz_helpers.h index 5f07fa4de935..468c39fb42d4 100644 --- a/tests/fuzz/fuzz_helpers.h +++ b/tests/fuzz/fuzz_helpers.h @@ -1,10 +1,10 @@ -/** +/* * Copyright (c) 2016-present, Facebook, Inc. * All rights reserved. * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). */ /** @@ -16,8 +16,14 @@ #include "fuzz.h" #include "xxhash.h" +#include "zstd.h" #include <stdint.h> #include <stdio.h> +#include <stdlib.h> + +#ifdef __cplusplus +extern "C" { +#endif #define MIN(a, b) ((a) < (b) ? (a) : (b)) #define MAX(a, b) ((a) > (b) ? (a) : (b)) @@ -34,6 +40,8 @@ __LINE__, FUZZ_QUOTE(cond), (msg)), \ abort())) #define FUZZ_ASSERT(cond) FUZZ_ASSERT_MSG((cond), ""); +#define FUZZ_ZASSERT(code) \ + FUZZ_ASSERT_MSG(!ZSTD_isError(code), ZSTD_getErrorName(code)) #if defined(__GNUC__) #define FUZZ_STATIC static __inline __attribute__((unused)) @@ -48,23 +56,37 @@ /** * Determininistically constructs a seed based on the fuzz input. - * Only looks at the first FUZZ_RNG_SEED_SIZE bytes of the input. + * Consumes up to the first FUZZ_RNG_SEED_SIZE bytes of the input. */ -FUZZ_STATIC uint32_t FUZZ_seed(const uint8_t *src, size_t size) { - size_t const toHash = MIN(FUZZ_RNG_SEED_SIZE, size); - return XXH32(src, toHash, 0); +FUZZ_STATIC uint32_t FUZZ_seed(uint8_t const **src, size_t* size) { + uint8_t const *data = *src; + size_t const toHash = MIN(FUZZ_RNG_SEED_SIZE, *size); + *size -= toHash; + *src += toHash; + return XXH32(data, toHash, 0); } #define FUZZ_rotl32(x, r) (((x) << (r)) | ((x) >> (32 - (r)))) + FUZZ_STATIC uint32_t FUZZ_rand(uint32_t *state) { - static const uint32_t prime1 = 2654435761U; - static const uint32_t prime2 = 2246822519U; - uint32_t rand32 = *state; - rand32 *= prime1; - rand32 += prime2; - rand32 = FUZZ_rotl32(rand32, 13); - *state = rand32; - return rand32 >> 5; + static const uint32_t prime1 = 2654435761U; + static const uint32_t prime2 = 2246822519U; + uint32_t rand32 = *state; + rand32 *= prime1; + rand32 += prime2; + rand32 = FUZZ_rotl32(rand32, 13); + *state = rand32; + return rand32 >> 5; } +/* Returns a random numer in the range [min, max]. */ +FUZZ_STATIC uint32_t FUZZ_rand32(uint32_t *state, uint32_t min, uint32_t max) { + uint32_t random = FUZZ_rand(state); + return min + (random % (max - min + 1)); +} + +#ifdef __cplusplus +} +#endif + #endif diff --git a/tests/fuzz/regression_driver.c b/tests/fuzz/regression_driver.c index eee5f0a2a2b2..2b714d29e9dd 100644 --- a/tests/fuzz/regression_driver.c +++ b/tests/fuzz/regression_driver.c @@ -1,10 +1,10 @@ -/** +/* * Copyright (c) 2016-present, Facebook, Inc. * All rights reserved. * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). */ #include "fuzz.h" @@ -29,9 +29,11 @@ int main(int argc, char const **argv) { #ifdef UTIL_HAS_CREATEFILELIST files = UTIL_createFileList(files, numFiles, &fileNamesBuf, &numFiles, kFollowLinks); - FUZZ_ASSERT(files); + if (!files) + numFiles = 0; #endif - + if (numFiles == 0) + fprintf(stderr, "WARNING: No files passed to %s\n", argv[0]); for (i = 0; i < numFiles; ++i) { char const *fileName = files[i]; size_t const fileSize = UTIL_getFileSize(fileName); @@ -39,7 +41,7 @@ int main(int argc, char const **argv) { FILE *file; /* Check that it is a regular file, and that the fileSize is valid */ - FUZZ_ASSERT_MSG(UTIL_isRegFile(fileName), fileName); + FUZZ_ASSERT_MSG(UTIL_isRegularFile(fileName), fileName); FUZZ_ASSERT_MSG(fileSize <= kMaxFileSize, fileName); /* Ensure we have a large enough buffer allocated */ if (fileSize > bufferSize) { diff --git a/tests/fuzz/simple_decompress.c b/tests/fuzz/simple_decompress.c index c22ad7c5301c..bba272c62256 100644 --- a/tests/fuzz/simple_decompress.c +++ b/tests/fuzz/simple_decompress.c @@ -1,10 +1,10 @@ -/** - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. +/* + * Copyright (c) 2016-present, Facebook, Inc. * All rights reserved. * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). */ /** @@ -24,7 +24,10 @@ static size_t bufSize = 0; int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { - size_t const neededBufSize = MAX(20 * size, (size_t)256 << 10); + size_t neededBufSize; + + FUZZ_seed(&src, &size); + neededBufSize = MAX(20 * size, (size_t)256 << 10); /* Allocate all buffers and contexts if not already allocated */ if (neededBufSize > bufSize) { @@ -39,7 +42,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) } ZSTD_decompressDCtx(dctx, rBuf, neededBufSize, src, size); -#ifndef STATEFULL_FUZZING +#ifndef STATEFUL_FUZZING ZSTD_freeDCtx(dctx); dctx = NULL; #endif return 0; diff --git a/tests/fuzz/simple_round_trip.c b/tests/fuzz/simple_round_trip.c index 703ea582630a..0921106dea8e 100644 --- a/tests/fuzz/simple_round_trip.c +++ b/tests/fuzz/simple_round_trip.c @@ -1,10 +1,10 @@ -/** +/* * Copyright (c) 2016-present, Facebook, Inc. * All rights reserved. * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). */ /** @@ -12,12 +12,14 @@ * compares the result with the original, and calls abort() on corruption. */ +#define ZSTD_STATIC_LINKING_ONLY + #include <stddef.h> #include <stdlib.h> #include <stdio.h> #include <string.h> #include "fuzz_helpers.h" -#include "zstd.h" +#include "zstd_helpers.h" static const int kMaxClevel = 19; @@ -32,21 +34,33 @@ static size_t roundTripTest(void *result, size_t resultCapacity, void *compressed, size_t compressedCapacity, const void *src, size_t srcSize) { - int const cLevel = FUZZ_rand(&seed) % kMaxClevel; - size_t const cSize = ZSTD_compressCCtx(cctx, compressed, compressedCapacity, - src, srcSize, cLevel); - if (ZSTD_isError(cSize)) { - fprintf(stderr, "Compression error: %s\n", ZSTD_getErrorName(cSize)); - return cSize; - } - return ZSTD_decompressDCtx(dctx, result, resultCapacity, compressed, cSize); + size_t cSize; + if (FUZZ_rand(&seed) & 1) { + ZSTD_inBuffer in = {src, srcSize, 0}; + ZSTD_outBuffer out = {compressed, compressedCapacity, 0}; + size_t err; + + ZSTD_CCtx_reset(cctx); + FUZZ_setRandomParameters(cctx, srcSize, &seed); + err = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end); + FUZZ_ZASSERT(err); + FUZZ_ASSERT(err == 0); + cSize = out.pos; + } else { + int const cLevel = FUZZ_rand(&seed) % kMaxClevel; + cSize = ZSTD_compressCCtx( + cctx, compressed, compressedCapacity, src, srcSize, cLevel); + } + FUZZ_ZASSERT(cSize); + return ZSTD_decompressDCtx(dctx, result, resultCapacity, compressed, cSize); } int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { - size_t const neededBufSize = ZSTD_compressBound(size); + size_t neededBufSize; - seed = FUZZ_seed(src, size); + seed = FUZZ_seed(&src, &size); + neededBufSize = ZSTD_compressBound(size); /* Allocate all buffers and contexts if not already allocated */ if (neededBufSize > bufSize) { @@ -69,11 +83,11 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { size_t const result = roundTripTest(rBuf, neededBufSize, cBuf, neededBufSize, src, size); - FUZZ_ASSERT_MSG(!ZSTD_isError(result), ZSTD_getErrorName(result)); + FUZZ_ZASSERT(result); FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size"); FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!"); } -#ifndef STATEFULL_FUZZING +#ifndef STATEFUL_FUZZING ZSTD_freeCCtx(cctx); cctx = NULL; ZSTD_freeDCtx(dctx); dctx = NULL; #endif diff --git a/tests/fuzz/stream_decompress.c b/tests/fuzz/stream_decompress.c index 778a426dec7c..7ad571221dff 100644 --- a/tests/fuzz/stream_decompress.c +++ b/tests/fuzz/stream_decompress.c @@ -1,10 +1,10 @@ -/** - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. +/* + * Copyright (c) 2016-present, Facebook, Inc. * All rights reserved. * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). */ /** @@ -20,7 +20,7 @@ #include "fuzz_helpers.h" #include "zstd.h" -static size_t const kBufSize = ZSTD_BLOCKSIZE_ABSOLUTEMAX; +static size_t const kBufSize = ZSTD_BLOCKSIZE_MAX; static ZSTD_DStream *dstream = NULL; static void* buf = NULL; @@ -51,7 +51,7 @@ static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size) int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { - seed = FUZZ_seed(src, size); + seed = FUZZ_seed(&src, &size); /* Allocate all buffers and contexts if not already allocated */ if (!buf) { @@ -78,7 +78,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) } error: -#ifndef STATEFULL_FUZZING +#ifndef STATEFUL_FUZZING ZSTD_freeDStream(dstream); dstream = NULL; #endif return 0; diff --git a/tests/fuzz/stream_round_trip.c b/tests/fuzz/stream_round_trip.c index 17c7dfdd29a7..72d70495f836 100644 --- a/tests/fuzz/stream_round_trip.c +++ b/tests/fuzz/stream_round_trip.c @@ -1,10 +1,10 @@ -/** +/* * Copyright (c) 2016-present, Facebook, Inc. * All rights reserved. * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). */ /** @@ -12,16 +12,16 @@ * compares the result with the original, and calls abort() on corruption. */ +#define ZSTD_STATIC_LINKING_ONLY + #include <stddef.h> #include <stdlib.h> #include <stdio.h> #include <string.h> #include "fuzz_helpers.h" -#include "zstd.h" +#include "zstd_helpers.h" -static const int kMaxClevel = 19; - -static ZSTD_CStream *cstream = NULL; +ZSTD_CCtx *cctx = NULL; static ZSTD_DCtx *dctx = NULL; static uint8_t* cBuf = NULL; static uint8_t* rBuf = NULL; @@ -30,93 +30,102 @@ static uint32_t seed; static ZSTD_outBuffer makeOutBuffer(uint8_t *dst, size_t capacity) { - ZSTD_outBuffer buffer = { dst, 0, 0 }; + ZSTD_outBuffer buffer = { dst, 0, 0 }; - FUZZ_ASSERT(capacity > 0); - buffer.size = (FUZZ_rand(&seed) % capacity) + 1; - FUZZ_ASSERT(buffer.size <= capacity); + FUZZ_ASSERT(capacity > 0); + buffer.size = (FUZZ_rand(&seed) % capacity) + 1; + FUZZ_ASSERT(buffer.size <= capacity); - return buffer; + return buffer; } static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size) { - ZSTD_inBuffer buffer = { *src, 0, 0 }; + ZSTD_inBuffer buffer = { *src, 0, 0 }; - FUZZ_ASSERT(*size > 0); - buffer.size = (FUZZ_rand(&seed) % *size) + 1; - FUZZ_ASSERT(buffer.size <= *size); - *src += buffer.size; - *size -= buffer.size; + FUZZ_ASSERT(*size > 0); + buffer.size = (FUZZ_rand(&seed) % *size) + 1; + FUZZ_ASSERT(buffer.size <= *size); + *src += buffer.size; + *size -= buffer.size; - return buffer; + return buffer; } static size_t compress(uint8_t *dst, size_t capacity, const uint8_t *src, size_t srcSize) { - int cLevel = FUZZ_rand(&seed) % kMaxClevel; size_t dstSize = 0; - FUZZ_ASSERT(!ZSTD_isError(ZSTD_initCStream(cstream, cLevel))); + ZSTD_CCtx_reset(cctx); + FUZZ_setRandomParameters(cctx, srcSize, &seed); while (srcSize > 0) { ZSTD_inBuffer in = makeInBuffer(&src, &srcSize); /* Mode controls the action. If mode == -1 we pick a new mode */ int mode = -1; while (in.pos < in.size) { - ZSTD_outBuffer out = makeOutBuffer(dst, capacity); - /* Previous action finished, pick a new mode. */ - if (mode == -1) mode = FUZZ_rand(&seed) % 10; - switch (mode) { - case 0: /* fall-though */ - case 1: /* fall-though */ - case 2: { - size_t const ret = ZSTD_flushStream(cstream, &out); - FUZZ_ASSERT_MSG(!ZSTD_isError(ret), ZSTD_getErrorName(ret)); - if (ret == 0) mode = -1; - break; - } - case 3: { - size_t ret = ZSTD_endStream(cstream, &out); - FUZZ_ASSERT_MSG(!ZSTD_isError(ret), ZSTD_getErrorName(ret)); - /* Reset the compressor when the frame is finished */ - if (ret == 0) { - cLevel = FUZZ_rand(&seed) % kMaxClevel; - ret = ZSTD_initCStream(cstream, cLevel); - FUZZ_ASSERT(!ZSTD_isError(ret)); + ZSTD_outBuffer out = makeOutBuffer(dst, capacity); + /* Previous action finished, pick a new mode. */ + if (mode == -1) mode = FUZZ_rand(&seed) % 10; + switch (mode) { + case 0: /* fall-though */ + case 1: /* fall-though */ + case 2: { + size_t const ret = + ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_flush); + FUZZ_ZASSERT(ret); + if (ret == 0) + mode = -1; + break; + } + case 3: { + size_t ret = + ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end); + FUZZ_ZASSERT(ret); + /* Reset the compressor when the frame is finished */ + if (ret == 0) { + ZSTD_CCtx_reset(cctx); + if ((FUZZ_rand(&seed) & 7) == 0) { + size_t const remaining = in.size - in.pos; + FUZZ_setRandomParameters(cctx, remaining, &seed); + } + mode = -1; + } + break; + } + default: { + size_t const ret = + ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_continue); + FUZZ_ZASSERT(ret); mode = -1; } - break; - } - default: { - size_t const ret = ZSTD_compressStream(cstream, &out, &in); - FUZZ_ASSERT_MSG(!ZSTD_isError(ret), ZSTD_getErrorName(ret)); - mode = -1; } - } - dst += out.pos; - dstSize += out.pos; - capacity -= out.pos; + dst += out.pos; + dstSize += out.pos; + capacity -= out.pos; } } for (;;) { + ZSTD_inBuffer in = {NULL, 0, 0}; ZSTD_outBuffer out = makeOutBuffer(dst, capacity); - size_t const ret = ZSTD_endStream(cstream, &out); - FUZZ_ASSERT_MSG(!ZSTD_isError(ret), ZSTD_getErrorName(ret)); + size_t const ret = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end); + FUZZ_ZASSERT(ret); dst += out.pos; dstSize += out.pos; capacity -= out.pos; - if (ret == 0) break; + if (ret == 0) + break; } return dstSize; } int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { - size_t const neededBufSize = ZSTD_compressBound(size) * 2; + size_t neededBufSize; - seed = FUZZ_seed(src, size); + seed = FUZZ_seed(&src, &size); + neededBufSize = ZSTD_compressBound(size) * 2; /* Allocate all buffers and contexts if not already allocated */ if (neededBufSize > bufSize) { @@ -127,9 +136,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) bufSize = neededBufSize; FUZZ_ASSERT(cBuf && rBuf); } - if (!cstream) { - cstream = ZSTD_createCStream(); - FUZZ_ASSERT(cstream); + if (!cctx) { + cctx = ZSTD_createCCtx(); + FUZZ_ASSERT(cctx); } if (!dctx) { dctx = ZSTD_createDCtx(); @@ -140,13 +149,13 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) size_t const cSize = compress(cBuf, neededBufSize, src, size); size_t const rSize = ZSTD_decompressDCtx(dctx, rBuf, neededBufSize, cBuf, cSize); - FUZZ_ASSERT_MSG(!ZSTD_isError(rSize), ZSTD_getErrorName(rSize)); + FUZZ_ZASSERT(rSize); FUZZ_ASSERT_MSG(rSize == size, "Incorrect regenerated size"); FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!"); } -#ifndef STATEFULL_FUZZING - ZSTD_freeCStream(cstream); cstream = NULL; +#ifndef STATEFUL_FUZZING + ZSTD_freeCCtx(cctx); cctx = NULL; ZSTD_freeDCtx(dctx); dctx = NULL; #endif return 0; diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c new file mode 100644 index 000000000000..602898479bc2 --- /dev/null +++ b/tests/fuzz/zstd_helpers.c @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + */ + +#define ZSTD_STATIC_LINKING_ONLY + +#include "zstd_helpers.h" +#include "fuzz_helpers.h" +#include "zstd.h" + +static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned value) +{ + FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, param, value)); +} + +static void setRand(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned min, + unsigned max, uint32_t *state) { + unsigned const value = FUZZ_rand32(state, min, max); + set(cctx, param, value); +} + +ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state) +{ + /* Select compression parameters */ + ZSTD_compressionParameters cParams; + cParams.windowLog = FUZZ_rand32(state, ZSTD_WINDOWLOG_MIN, 15); + cParams.hashLog = FUZZ_rand32(state, ZSTD_HASHLOG_MIN, 15); + cParams.chainLog = FUZZ_rand32(state, ZSTD_CHAINLOG_MIN, 16); + cParams.searchLog = FUZZ_rand32(state, ZSTD_SEARCHLOG_MIN, 9); + cParams.searchLength = FUZZ_rand32(state, ZSTD_SEARCHLENGTH_MIN, + ZSTD_SEARCHLENGTH_MAX); + cParams.targetLength = FUZZ_rand32(state, ZSTD_TARGETLENGTH_MIN, + ZSTD_TARGETLENGTH_MAX); + cParams.strategy = FUZZ_rand32(state, ZSTD_fast, ZSTD_btultra); + return ZSTD_adjustCParams(cParams, srcSize, 0); +} + +ZSTD_frameParameters FUZZ_randomFParams(uint32_t *state) +{ + /* Select frame parameters */ + ZSTD_frameParameters fParams; + fParams.contentSizeFlag = FUZZ_rand32(state, 0, 1); + fParams.checksumFlag = FUZZ_rand32(state, 0, 1); + fParams.noDictIDFlag = FUZZ_rand32(state, 0, 1); + return fParams; +} + +ZSTD_parameters FUZZ_randomParams(size_t srcSize, uint32_t *state) +{ + ZSTD_parameters params; + params.cParams = FUZZ_randomCParams(srcSize, state); + params.fParams = FUZZ_randomFParams(state); + return params; +} + +void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state) +{ + ZSTD_compressionParameters cParams = FUZZ_randomCParams(srcSize, state); + set(cctx, ZSTD_p_windowLog, cParams.windowLog); + set(cctx, ZSTD_p_hashLog, cParams.hashLog); + set(cctx, ZSTD_p_chainLog, cParams.chainLog); + set(cctx, ZSTD_p_searchLog, cParams.searchLog); + set(cctx, ZSTD_p_minMatch, cParams.searchLength); + set(cctx, ZSTD_p_targetLength, cParams.targetLength); + set(cctx, ZSTD_p_compressionStrategy, cParams.strategy); + /* Select frame parameters */ + setRand(cctx, ZSTD_p_contentSizeFlag, 0, 1, state); + setRand(cctx, ZSTD_p_checksumFlag, 0, 1, state); + setRand(cctx, ZSTD_p_dictIDFlag, 0, 1, state); + /* Select long distance matchig parameters */ + setRand(cctx, ZSTD_p_enableLongDistanceMatching, 0, 1, state); + setRand(cctx, ZSTD_p_ldmHashLog, ZSTD_HASHLOG_MIN, 16, state); + setRand(cctx, ZSTD_p_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN, + ZSTD_LDM_MINMATCH_MAX, state); + setRand(cctx, ZSTD_p_ldmBucketSizeLog, 0, ZSTD_LDM_BUCKETSIZELOG_MAX, + state); + setRand(cctx, ZSTD_p_ldmHashEveryLog, 0, + ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN, state); +} diff --git a/tests/fuzz/zstd_helpers.h b/tests/fuzz/zstd_helpers.h new file mode 100644 index 000000000000..3856bebecf7e --- /dev/null +++ b/tests/fuzz/zstd_helpers.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + */ + +/** + * Helper functions for fuzzing. + */ + +#ifndef ZSTD_HELPERS_H +#define ZSTD_HELPERS_H + +#include "zstd.h" +#include <stdint.h> + +#ifdef __cplusplus +extern "C" { +#endif + +void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state); + +ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state); +ZSTD_frameParameters FUZZ_randomFParams(uint32_t *state); +ZSTD_parameters FUZZ_randomParams(size_t srcSize, uint32_t *state); + + +#ifdef __cplusplus +} +#endif + +#endif /* ZSTD_HELPERS_H */ diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 0c13a6e488ad..76df77af9015 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1,10 +1,11 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2015-present, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. */ @@ -509,8 +510,7 @@ static int basicUnitTests(U32 seed, double compressibility) /* only use the first half so we don't push against size limit of compressedBuffer */ size_t const segSize = (CNBuffSize / 2) / segs; for (i = 0; i < segs; i++) { - CHECK_V(r, - ZSTD_compress( + CHECK_V(r, ZSTD_compress( (BYTE *)compressedBuffer + off, CNBuffSize - off, (BYTE *)CNBuffer + segSize * i, segSize, 5)); @@ -598,10 +598,10 @@ static int basicUnitTests(U32 seed, double compressibility) } DISPLAYLEVEL(4, "test%3i : decompress with static DDict : ", testNb++); - { size_t const ddictBufferSize = ZSTD_estimateDDictSize(dictSize, 0); + { size_t const ddictBufferSize = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy); void* ddictBuffer = malloc(ddictBufferSize); if (ddictBuffer == NULL) goto _output_error; - { ZSTD_DDict* const ddict = ZSTD_initStaticDDict(ddictBuffer, ddictBufferSize, CNBuffer, dictSize, 0); + { ZSTD_DDict* const ddict = ZSTD_initStaticDDict(ddictBuffer, ddictBufferSize, CNBuffer, dictSize, ZSTD_dlm_byCopy); size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict); if (r != CNBuffSize - dictSize) goto _output_error; } @@ -687,14 +687,14 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "test%3i : estimate CDict size : ", testNb++); { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); - size_t const estimatedSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, 1 /*byReference*/); + size_t const estimatedSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byRef); DISPLAYLEVEL(4, "OK : %u \n", (U32)estimatedSize); } DISPLAYLEVEL(4, "test%3i : compress with CDict ", testNb++); { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, - 1 /* byReference */, ZSTD_dm_auto, + ZSTD_dlm_byRef, ZSTD_dm_auto, cParams, ZSTD_defaultCMem); DISPLAYLEVEL(4, "(size : %u) : ", (U32)ZSTD_sizeof_CDict(cdict)); cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, @@ -720,12 +720,12 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "test%3i : compress with static CDict : ", testNb++); { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); - size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, 0); + size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); void* const cdictBuffer = malloc(cdictSize); if (cdictBuffer==NULL) goto _output_error; { ZSTD_CDict* const cdict = ZSTD_initStaticCDict(cdictBuffer, cdictSize, dictBuffer, dictSize, - 0 /* by Reference */, ZSTD_dm_auto, + ZSTD_dlm_byCopy, ZSTD_dm_auto, cParams); if (cdict == NULL) { DISPLAY("ZSTD_initStaticCDict failed "); @@ -745,7 +745,7 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++); { ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ }; ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); - ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 1 /*byRef*/, ZSTD_dm_auto, cParams, ZSTD_defaultCMem); + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dm_auto, cParams, ZSTD_defaultCMem); cSize = ZSTD_compress_usingCDict_advanced(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, cdict, fParams); ZSTD_freeCDict(cdict); @@ -796,7 +796,7 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a good dictionary : ", testNb++); { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); - ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 1 /*byRef*/, ZSTD_dm_fullDict, cParams, ZSTD_defaultCMem); + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dm_fullDict, cParams, ZSTD_defaultCMem); if (cdict==NULL) goto _output_error; ZSTD_freeCDict(cdict); } @@ -804,12 +804,32 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a rawContent (must fail) : ", testNb++); { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); - ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, 1 /*byRef*/, ZSTD_dm_fullDict, cParams, ZSTD_defaultCMem); + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dm_fullDict, cParams, ZSTD_defaultCMem); if (cdict!=NULL) goto _output_error; ZSTD_freeCDict(cdict); } DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(4, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_auto should fail : ", testNb++); + { + size_t ret; + MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY); + ret = ZSTD_CCtx_loadDictionary_advanced( + cctx, (const char*)dictBuffer+2, dictSize-2, ZSTD_dlm_byRef, ZSTD_dm_auto); + if (!ZSTD_isError(ret)) goto _output_error; + } + DISPLAYLEVEL(4, "OK \n"); + + DISPLAYLEVEL(4, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_rawContent should pass : ", testNb++); + { + size_t ret; + MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY); + ret = ZSTD_CCtx_loadDictionary_advanced( + cctx, (const char*)dictBuffer+2, dictSize-2, ZSTD_dlm_byRef, ZSTD_dm_rawContent); + if (ZSTD_isError(ret)) goto _output_error; + } + DISPLAYLEVEL(4, "OK \n"); + ZSTD_freeCCtx(cctx); free(dictBuffer); free(samplesSizes); @@ -897,6 +917,43 @@ static int basicUnitTests(U32 seed, double compressibility) ZSTD_freeCCtx(cctx); } + /* custom formats tests */ + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + size_t const inputSize = CNBuffSize / 2; /* won't cause pb with small dict size */ + + /* basic block compression */ + DISPLAYLEVEL(4, "test%3i : magic-less format test : ", testNb++); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_format, ZSTD_f_zstd1_magicless) ); + { ZSTD_inBuffer in = { CNBuffer, inputSize, 0 }; + ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 }; + size_t const result = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end); + if (result != 0) goto _output_error; + if (in.pos != in.size) goto _output_error; + cSize = out.pos; + } + DISPLAYLEVEL(4, "OK (compress : %u -> %u bytes)\n", (U32)inputSize, (U32)cSize); + + DISPLAYLEVEL(4, "test%3i : decompress normally (should fail) : ", testNb++); + { size_t const decodeResult = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); + if (ZSTD_getErrorCode(decodeResult) != ZSTD_error_prefix_unknown) goto _output_error; + DISPLAYLEVEL(4, "OK : %s \n", ZSTD_getErrorName(decodeResult)); + } + + DISPLAYLEVEL(4, "test%3i : decompress with magic-less instruction : ", testNb++); + ZSTD_DCtx_reset(dctx); + CHECK( ZSTD_DCtx_setFormat(dctx, ZSTD_f_zstd1_magicless) ); + { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 }; + ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 }; + size_t const result = ZSTD_decompress_generic(dctx, &out, &in); + if (result != 0) goto _output_error; + if (in.pos != in.size) goto _output_error; + if (out.pos != inputSize) goto _output_error; + DISPLAYLEVEL(4, "OK : regenerated %u bytes \n", (U32)out.pos); + } + + ZSTD_freeCCtx(cctx); + } + /* block API tests */ { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); static const size_t dictSize = 65 KB; @@ -940,8 +997,8 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "OK \n"); ZSTD_freeCCtx(cctx); - ZSTD_freeDCtx(dctx); } + ZSTD_freeDCtx(dctx); /* long rle test */ { size_t sampleSize = 0; @@ -1334,7 +1391,6 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD } CHECK_Z( ZSTD_copyCCtx(ctx, refCtx, 0) ); } - ZSTD_setCCtxParameter(ctx, ZSTD_p_forceWindow, FUZ_rand(&lseed) & 1); { U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2; U32 n; @@ -1364,6 +1420,16 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD } /* streaming decompression test */ + /* ensure memory requirement is good enough (should always be true) */ + { ZSTD_frameHeader zfh; + CHECK( ZSTD_getFrameHeader(&zfh, cBuffer, ZSTD_frameHeaderSize_max), + "ZSTD_getFrameHeader(): error retrieving frame information"); + { size_t const roundBuffSize = ZSTD_decodingBufferSize_min(zfh.windowSize, zfh.frameContentSize); + CHECK_Z(roundBuffSize); + CHECK((roundBuffSize > totalTestSize) && (zfh.frameContentSize!=ZSTD_CONTENTSIZE_UNKNOWN), + "ZSTD_decodingBufferSize_min() requires more memory (%u) than necessary (%u)", + (U32)roundBuffSize, (U32)totalTestSize ); + } } if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */ CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, dict, dictSize) ); totalCSize = 0; diff --git a/tests/gzip/Makefile b/tests/gzip/Makefile index b02fb693f329..40a0ba97d2b3 100644 --- a/tests/gzip/Makefile +++ b/tests/gzip/Makefile @@ -1,10 +1,10 @@ # ################################################################ -# Copyright (c) 2017-present, Yann Collet, Facebook, Inc. +# Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. An additional grant -# of patent rights can be found in the PATENTS file in the same directory. +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). # ################################################################ PRGDIR = ../../programs @@ -12,7 +12,7 @@ VOID = /dev/null export PATH := .:$(PATH) .PHONY: all -#all: test-gzip-env +#all: test-gzip-env all: test-helin-segv test-hufts test-keep test-list test-memcpy-abuse test-mixed all: test-null-suffix-clobber test-stdin test-trailing-nul test-unpack-invalid all: test-zdiff test-zgrep-context test-zgrep-f test-zgrep-signal test-znew-k test-z-suffix diff --git a/tests/invalidDictionaries.c b/tests/invalidDictionaries.c index 83fe439d4c43..b23db036f489 100644 --- a/tests/invalidDictionaries.c +++ b/tests/invalidDictionaries.c @@ -5,6 +5,7 @@ * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. */ #include <stddef.h> diff --git a/tests/legacy.c b/tests/legacy.c index 962b2c9c3c9d..46a8206c4422 100644 --- a/tests/legacy.c +++ b/tests/legacy.c @@ -5,6 +5,7 @@ * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. */ /* diff --git a/tests/longmatch.c b/tests/longmatch.c index ef79337f56d5..ed3861571d9b 100644 --- a/tests/longmatch.c +++ b/tests/longmatch.c @@ -1,10 +1,11 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2017-present, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. */ diff --git a/tests/namespaceTest.c b/tests/namespaceTest.c index 6f6c74fd63d9..5b7095f67c80 100644 --- a/tests/namespaceTest.c +++ b/tests/namespaceTest.c @@ -5,6 +5,7 @@ * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. */ diff --git a/tests/paramgrill.c b/tests/paramgrill.c index ed13e1dacca8..317ec461ca91 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -1,10 +1,11 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2015-present, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. */ @@ -390,8 +391,8 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed); double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed); - size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize_advanced(params); - size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize_advanced(winners[cLevel].params); + size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(params); + size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(winners[cLevel].params); double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed); double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed); diff --git a/tests/playTests.sh b/tests/playTests.sh index bc8584e7a9f0..bc021648cfcb 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -13,11 +13,16 @@ roundTripTest() { cLevel="$2" proba="" fi + if [ -n "$4" ]; then + dLevel="$4" + else + dLevel="$cLevel" + fi rm -f tmp1 tmp2 - $ECHO "roundTripTest: ./datagen $1 $proba | $ZSTD -v$cLevel | $ZSTD -d" + $ECHO "roundTripTest: ./datagen $1 $proba | $ZSTD -v$cLevel | $ZSTD -d$dLevel" ./datagen $1 $proba | $MD5SUM > tmp1 - ./datagen $1 $proba | $ZSTD --ultra -v$cLevel | $ZSTD -d | $MD5SUM > tmp2 + ./datagen $1 $proba | $ZSTD --ultra -v$cLevel | $ZSTD -d$dLevel | $MD5SUM > tmp2 $DIFF -q tmp1 tmp2 } @@ -29,12 +34,17 @@ fileRoundTripTest() { local_c="$2" local_p="" fi + if [ -n "$4" ]; then + local_d="$4" + else + local_d="$local_c" + fi rm -f tmp.zstd tmp.md5.1 tmp.md5.2 - $ECHO "fileRoundTripTest: ./datagen $1 $local_p > tmp && $ZSTD -v$local_c -c tmp | $ZSTD -d" + $ECHO "fileRoundTripTest: ./datagen $1 $local_p > tmp && $ZSTD -v$local_c -c tmp | $ZSTD -d$local_d" ./datagen $1 $local_p > tmp - cat tmp | $MD5SUM > tmp.md5.1 - $ZSTD --ultra -v$local_c -c tmp | $ZSTD -d | $MD5SUM > tmp.md5.2 + < tmp $MD5SUM > tmp.md5.1 + $ZSTD --ultra -v$local_c -c tmp | $ZSTD -d$local_d | $MD5SUM > tmp.md5.2 $DIFF -q tmp.md5.1 tmp.md5.2 } @@ -45,7 +55,6 @@ then fi isWindows=false -ECHO="echo -e" INTOVOID="/dev/null" case "$OS" in Windows*) @@ -66,6 +75,11 @@ case "$UNAME" in SunOS) DIFF="gdiff" ;; esac +ECHO="echo -e" +case "$UNAME" in + Darwin) ECHO="echo" ;; +esac + $ECHO "\nStarting playTests.sh isWindows=$isWindows ZSTD='$ZSTD'" [ -n "$ZSTD" ] || die "ZSTD variable must be defined!" @@ -161,6 +175,8 @@ roundTripTest -g512K roundTripTest -g512K " --zstd=slen=3,tlen=48,strat=6" roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6" roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6" +roundTripTest -g512K " --long --zstd=ldmHashLog=20,ldmSearchLength=64,ldmBucketSizeLog=1,ldmHashEveryLog=7" +roundTripTest -g512K " --long --zstd=ldmhlog=20,ldmslen=64,ldmblog=1,ldmhevery=7" roundTripTest -g512K 19 @@ -291,8 +307,10 @@ $ECHO "- Create dictionary with wrong dictID parameter order (must fail)" $ZSTD --train *.c ../programs/*.c --dictID -o 1 tmpDict1 && die "wrong order : --dictID must be followed by argument " $ECHO "- Create dictionary with size limit" $ZSTD --train *.c ../programs/*.c -o tmpDict2 --maxdict=4K -v +$ECHO "- Create dictionary with small size limit" +$ZSTD --train *.c ../programs/*.c -o tmpDict3 --maxdict=1K -v $ECHO "- Create dictionary with wrong parameter order (must fail)" -$ZSTD --train *.c ../programs/*.c -o tmpDict2 --maxdict -v 4K && die "wrong order : --maxdict must be followed by argument " +$ZSTD --train *.c ../programs/*.c -o tmpDict3 --maxdict -v 4K && die "wrong order : --maxdict must be followed by argument " $ECHO "- Compress without dictID" $ZSTD -f tmp -D tmpDict1 --no-dictID $ZSTD -d tmp.zst -D tmpDict -fo result @@ -551,6 +569,15 @@ roundTripTest -g516K 19 # btopt fileRoundTripTest -g500K +$ECHO "\n**** zstd long distance matching round-trip tests **** " +roundTripTest -g0 "2 --long" +roundTripTest -g1000K "1 --long" +roundTripTest -g517K "6 --long" +roundTripTest -g516K "16 --long" +roundTripTest -g518K "19 --long" +fileRoundTripTest -g5M "3 --long" + + if [ -n "$hasMT" ] then $ECHO "\n**** zstdmt round-trip tests **** " @@ -558,6 +585,9 @@ then roundTripTest -g8M "3 -T2" roundTripTest -g8000K "2 --threads=2" fileRoundTripTest -g4M "19 -T2 -B1M" + + $ECHO "\n**** zstdmt long distance matching round-trip tests **** " + roundTripTest -g8M "3 --long -T2" else $ECHO "\n**** no multithreading, skipping zstdmt tests **** " fi @@ -568,7 +598,6 @@ $ECHO "\n**** zstd --list/-l single frame tests ****" ./datagen > tmp1 ./datagen > tmp2 ./datagen > tmp3 -./datagen > tmp4 $ZSTD tmp* $ZSTD -l *.zst $ZSTD -lv *.zst @@ -577,9 +606,7 @@ $ZSTD --list -v *.zst $ECHO "\n**** zstd --list/-l multiple frame tests ****" cat tmp1.zst tmp2.zst > tmp12.zst -cat tmp3.zst tmp4.zst > tmp34.zst -cat tmp12.zst tmp34.zst > tmp1234.zst -cat tmp12.zst tmp4.zst > tmp124.zst +cat tmp12.zst tmp3.zst > tmp123.zst $ZSTD -l *.zst $ZSTD -lv *.zst $ZSTD --list *.zst @@ -589,7 +616,7 @@ $ECHO "\n**** zstd --list/-l error detection tests ****" ! $ZSTD -l tmp1 tmp1.zst ! $ZSTD --list tmp* ! $ZSTD -lv tmp1* -! $ZSTD --list -v tmp2 tmp23.zst +! $ZSTD --list -v tmp2 tmp12.zst $ECHO "\n**** zstd --list/-l test with null files ****" ./datagen -g0 > tmp5 @@ -612,47 +639,70 @@ $ZSTD -lv tmp1.zst rm tmp* +$ECHO "\n**** zstd long distance matching tests **** " +roundTripTest -g0 " --long" +roundTripTest -g9M "2 --long" +# Test parameter parsing +roundTripTest -g1M -P50 "1 --long=29" " --memory=512MB" +roundTripTest -g1M -P50 "1 --long=29 --zstd=wlog=28" " --memory=256MB" +roundTripTest -g1M -P50 "1 --long=29" " --long=28 --memory=512MB" +roundTripTest -g1M -P50 "1 --long=29" " --zstd=wlog=28 --memory=512MB" + + if [ "$1" != "--test-large-data" ]; then $ECHO "Skipping large data tests" exit 0 fi +$ECHO "\n**** large files tests **** " + roundTripTest -g270000000 1 -roundTripTest -g270000000 2 -roundTripTest -g270000000 3 +roundTripTest -g250000000 2 +roundTripTest -g230000000 3 roundTripTest -g140000000 -P60 4 -roundTripTest -g140000000 -P60 5 -roundTripTest -g140000000 -P60 6 +roundTripTest -g130000000 -P62 5 +roundTripTest -g120000000 -P65 6 roundTripTest -g70000000 -P70 7 -roundTripTest -g70000000 -P70 8 -roundTripTest -g70000000 -P70 9 +roundTripTest -g60000000 -P71 8 +roundTripTest -g50000000 -P73 9 roundTripTest -g35000000 -P75 10 -roundTripTest -g35000000 -P75 11 -roundTripTest -g35000000 -P75 12 +roundTripTest -g30000000 -P76 11 +roundTripTest -g25000000 -P78 12 roundTripTest -g18000013 -P80 13 roundTripTest -g18000014 -P80 14 -roundTripTest -g18000015 -P80 15 -roundTripTest -g18000016 -P80 16 -roundTripTest -g18000017 -P80 17 +roundTripTest -g18000015 -P81 15 +roundTripTest -g18000016 -P84 16 +roundTripTest -g18000017 -P88 17 roundTripTest -g18000018 -P94 18 -roundTripTest -g18000019 -P94 19 +roundTripTest -g18000019 -P96 19 -roundTripTest -g68000020 -P99 20 -roundTripTest -g6000000000 -P99 1 +roundTripTest -g5000000000 -P99 1 fileRoundTripTest -g4193M -P99 1 + +$ECHO "\n**** zstd long, long distance matching round-trip tests **** " +roundTripTest -g270000000 "1 --long" +roundTripTest -g130000000 -P60 "5 --long" +roundTripTest -g35000000 -P70 "8 --long" +roundTripTest -g18000001 -P80 "18 --long" +# Test large window logs +roundTripTest -g700M -P50 "1 --long=29" +roundTripTest -g600M -P50 "1 --long --zstd=wlog=29,clog=28" + + if [ -n "$hasMT" ] then $ECHO "\n**** zstdmt long round-trip tests **** " - roundTripTest -g99000000 -P99 "20 -T2" - roundTripTest -g6000000000 -P99 "1 -T2" - roundTripTest -g1500000000 -P97 "1 -T999" - fileRoundTripTest -g4195M -P98 " -T0" + roundTripTest -g80000000 -P99 "19 -T2" " " + roundTripTest -g5000000000 -P99 "1 -T2" " " + roundTripTest -g500000000 -P97 "1 -T999" " " + fileRoundTripTest -g4103M -P98 " -T0" " " + roundTripTest -g400000000 -P97 "1 --long=24 -T2" " " else $ECHO "\n**** no multithreading, skipping zstdmt tests **** " fi diff --git a/tests/poolTests.c b/tests/poolTests.c index f3d5c382ae57..00ee830154c9 100644 --- a/tests/poolTests.c +++ b/tests/poolTests.c @@ -5,6 +5,7 @@ * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. */ diff --git a/tests/roundTripCrash.c b/tests/roundTripCrash.c index 0b478f6cab3e..180fa9b6f6dc 100644 --- a/tests/roundTripCrash.c +++ b/tests/roundTripCrash.c @@ -5,6 +5,7 @@ * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. */ /* @@ -20,9 +21,12 @@ #include <stddef.h> /* size_t */ #include <stdlib.h> /* malloc, free, exit */ #include <stdio.h> /* fprintf */ +#include <string.h> /* strcmp */ #include <sys/types.h> /* stat */ #include <sys/stat.h> /* stat */ #include "xxhash.h" + +#define ZSTD_STATIC_LINKING_ONLY #include "zstd.h" /*=========================================== @@ -30,6 +34,24 @@ *==========================================*/ #define MIN(a,b) ( (a) < (b) ? (a) : (b) ) +static void crash(int errorCode){ + /* abort if AFL/libfuzzer, exit otherwise */ + #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION /* could also use __AFL_COMPILER */ + abort(); + #else + exit(errorCode); + #endif +} + +#define CHECK_Z(f) { \ + size_t const err = f; \ + if (ZSTD_isError(err)) { \ + fprintf(stderr, \ + "Error=> %s: %s", \ + #f, ZSTD_getErrorName(err)); \ + crash(1); \ +} } + /** roundTripTest() : * Compresses `srcBuff` into `compressedBuff`, * then decompresses `compressedBuff` into `resultBuff`. @@ -54,6 +76,38 @@ static size_t roundTripTest(void* resultBuff, size_t resultBuffCapacity, return ZSTD_decompress(resultBuff, resultBuffCapacity, compressedBuff, cSize); } +/** cctxParamRoundTripTest() : + * Same as roundTripTest() except allows experimenting with ZSTD_CCtx_params. */ +static size_t cctxParamRoundTripTest(void* resultBuff, size_t resultBuffCapacity, + void* compressedBuff, size_t compressedBuffCapacity, + const void* srcBuff, size_t srcBuffSize) +{ + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_CCtx_params* const cctxParams = ZSTD_createCCtxParams(); + ZSTD_inBuffer inBuffer = { srcBuff, srcBuffSize, 0 }; + ZSTD_outBuffer outBuffer = {compressedBuff, compressedBuffCapacity, 0 }; + + static const int maxClevel = 19; + size_t const hashLength = MIN(128, srcBuffSize); + unsigned const h32 = XXH32(srcBuff, hashLength, 0); + int const cLevel = h32 % maxClevel; + + /* Set parameters */ + CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_compressionLevel, cLevel) ); + CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_nbThreads, 2) ); + CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_overlapSizeLog, 5) ); + + + /* Apply parameters */ + CHECK_Z( ZSTD_CCtx_setParametersUsingCCtxParams(cctx, cctxParams) ); + + CHECK_Z (ZSTD_compress_generic(cctx, &outBuffer, &inBuffer, ZSTD_e_end) ); + + ZSTD_freeCCtxParams(cctxParams); + ZSTD_freeCCtx(cctx); + + return ZSTD_decompress(resultBuff, resultBuffCapacity, compressedBuff, outBuffer.pos); +} static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize) { @@ -68,16 +122,7 @@ static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize return pos; } -static void crash(int errorCode){ - /* abort if AFL/libfuzzer, exit otherwise */ - #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION /* could also use __AFL_COMPILER */ - abort(); - #else - exit(errorCode); - #endif -} - -static void roundTripCheck(const void* srcBuff, size_t srcBuffSize) +static void roundTripCheck(const void* srcBuff, size_t srcBuffSize, int testCCtxParams) { size_t const cBuffSize = ZSTD_compressBound(srcBuffSize); void* cBuff = malloc(cBuffSize); @@ -88,7 +133,9 @@ static void roundTripCheck(const void* srcBuff, size_t srcBuffSize) exit (1); } - { size_t const result = roundTripTest(rBuff, cBuffSize, cBuff, cBuffSize, srcBuff, srcBuffSize); + { size_t const result = testCCtxParams ? + cctxParamRoundTripTest(rBuff, cBuffSize, cBuff, cBuffSize, srcBuff, srcBuffSize) + : roundTripTest(rBuff, cBuffSize, cBuff, cBuffSize, srcBuff, srcBuffSize); if (ZSTD_isError(result)) { fprintf(stderr, "roundTripTest error : %s \n", ZSTD_getErrorName(result)); crash(1); @@ -162,7 +209,7 @@ static void loadFile(void* buffer, const char* fileName, size_t fileSize) } -static void fileCheck(const char* fileName) +static void fileCheck(const char* fileName, int testCCtxParams) { size_t const fileSize = getFileSize(fileName); void* buffer = malloc(fileSize); @@ -171,16 +218,24 @@ static void fileCheck(const char* fileName) exit(4); } loadFile(buffer, fileName, fileSize); - roundTripCheck(buffer, fileSize); + roundTripCheck(buffer, fileSize, testCCtxParams); free (buffer); } int main(int argCount, const char** argv) { + int argNb = 1; + int testCCtxParams = 0; if (argCount < 2) { fprintf(stderr, "Error : no argument : need input file \n"); exit(9); } - fileCheck(argv[1]); + + if (!strcmp(argv[argNb], "--cctxParams")) { + testCCtxParams = 1; + argNb++; + } + + fileCheck(argv[argNb], testCCtxParams); fprintf(stderr, "no pb detected\n"); return 0; } diff --git a/tests/symbols.c b/tests/symbols.c index f08542dbf669..c0bed2e5d96e 100644 --- a/tests/symbols.c +++ b/tests/symbols.c @@ -5,6 +5,7 @@ * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. */ diff --git a/tests/test-zstd-speed.py b/tests/test-zstd-speed.py index 56108a5cae4c..1096d5e4e224 100755 --- a/tests/test-zstd-speed.py +++ b/tests/test-zstd-speed.py @@ -1,13 +1,13 @@ #! /usr/bin/env python3 -# +# ################################################################ # Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. # All rights reserved. # -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. An additional grant -# of patent rights can be found in the PATENTS file in the same directory. -# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ########################################################################## # Limitations: # - doesn't support filenames with spaces diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py index a5a713009a93..f2deac1f28da 100755 --- a/tests/test-zstd-versions.py +++ b/tests/test-zstd-versions.py @@ -1,14 +1,14 @@ #!/usr/bin/env python3 """Test zstd interoperability between versions""" -# +# ################################################################ # Copyright (c) 2016-present, Yann Collet, Facebook, Inc. # All rights reserved. # -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. An additional grant -# of patent rights can be found in the PATENTS file in the same directory. -# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ################################################################ import filecmp import glob diff --git a/tests/zbufftest.c b/tests/zbufftest.c index fe08fdab5bed..e64c886a5e75 100644 --- a/tests/zbufftest.c +++ b/tests/zbufftest.c @@ -1,10 +1,11 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2015-present, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. */ diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index dd044342ea80..8b4c836940a6 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -5,6 +5,7 @@ * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. */ @@ -95,6 +96,15 @@ unsigned int FUZ_rand(unsigned int* seedPtr) return rand32 >> 5; } +#define CHECK_Z(f) { \ + size_t const err = f; \ + if (ZSTD_isError(err)) { \ + DISPLAY("Error => %s : %s ", \ + #f, ZSTD_getErrorName(err)); \ + DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); \ + goto _output_error; \ +} } + /*====================================================== * Basic Unit tests @@ -149,6 +159,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo U32 testNb = 1; ZSTD_CStream* zc = ZSTD_createCStream_advanced(customMem); ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem); + ZSTDMT_CCtx* mtctx = ZSTDMT_createCCtx(2); + ZSTD_inBuffer inBuff, inBuff2; ZSTD_outBuffer outBuff; buffer_t dictionary = g_nullBuffer; @@ -163,8 +175,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed); /* Create dictionary */ - MEM_STATIC_ASSERT(COMPRESSIBLE_NOISE_LENGTH >= 4 MB); - dictionary = FUZ_createDictionary(CNBuffer, 4 MB, 4 KB, 40 KB); + DISPLAYLEVEL(3, "creating dictionary for unit tests \n"); + dictionary = FUZ_createDictionary(CNBuffer, CNBufferSize / 2, 8 KB, 40 KB); if (!dictionary.start) { DISPLAY("Error creating dictionary, aborting \n"); goto _output_error; @@ -178,16 +190,14 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo /* Basic compression test */ DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); - { size_t const r = ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1); - if (ZSTD_isError(r)) goto _output_error; } + CHECK_Z( ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1) ); outBuff.dst = (char*)(compressedBuffer)+cSize; outBuff.size = compressedBufferSize; outBuff.pos = 0; inBuff.src = CNBuffer; inBuff.size = CNBufferSize; inBuff.pos = 0; - { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff); - if (ZSTD_isError(r)) goto _output_error; } + CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ { size_t const r = ZSTD_endStream(zc, &outBuff); if (r != 0) goto _output_error; } /* error, or some data not flushed */ @@ -197,11 +207,11 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo /* context size functions */ DISPLAYLEVEL(3, "test%3i : estimate CStream size : ", testNb++); { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictSize); - size_t const s = ZSTD_estimateCStreamSize_advanced(cParams) - /* uses ZSTD_initCStream_usingDict() */ - + ZSTD_estimateCDictSize_advanced(dictSize, cParams, 0); - if (ZSTD_isError(s)) goto _output_error; - DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s); + size_t const cstreamSize = ZSTD_estimateCStreamSize_usingCParams(cParams); + size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); /* uses ZSTD_initCStream_usingDict() */ + if (ZSTD_isError(cstreamSize)) goto _output_error; + if (ZSTD_isError(cdictSize)) goto _output_error; + DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)(cstreamSize + cdictSize)); } DISPLAYLEVEL(3, "test%3i : check actual CStream size : ", testNb++); @@ -222,8 +232,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo /* skippable frame test */ DISPLAYLEVEL(3, "test%3i : decompress skippable frame : ", testNb++); - if (ZSTD_isError( ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize) )) - goto _output_error; + CHECK_Z( ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize) ); inBuff.src = compressedBuffer; inBuff.size = cSize; inBuff.pos = 0; @@ -241,8 +250,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo inBuff2 = inBuff; DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize); - { size_t const r = ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1000000000); /* large limit */ - if (ZSTD_isError(r)) goto _output_error; } + CHECK_Z( ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1000000000) ); /* large limit */ { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff); if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */ @@ -275,7 +283,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo DISPLAYLEVEL(5, " (windowSize : %u) ", (U32)fhi.windowSize); { size_t const s = ZSTD_estimateDStreamSize(fhi.windowSize) /* uses ZSTD_initDStream_usingDict() */ - + ZSTD_estimateDDictSize(dictSize, 0); + + ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy); if (ZSTD_isError(s)) goto _output_error; DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s); } } @@ -332,8 +340,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo inBuff.src = CNBuffer; inBuff.size = CNBufferSize; inBuff.pos = 0; - { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff); - if (ZSTD_isError(r)) goto _output_error; } + CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ { size_t const r = ZSTD_endStream(zc, &outBuff); if (r != 0) goto _output_error; } /* error, or some data not flushed */ @@ -350,8 +357,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo inBuff.src = CNBuffer; inBuff.size = CNBufferSize; inBuff.pos = 0; - { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff); - if (ZSTD_isError(r)) goto _output_error; } + CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ { size_t const r = ZSTD_endStream(zc, &outBuff); if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */ @@ -373,8 +379,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo outBuff.size = ZSTD_compressBound(inSize); outBuff.pos = 0; DISPLAYLEVEL(5, "compress1 "); - { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff); - if (ZSTD_isError(r)) goto _output_error; } + CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ DISPLAYLEVEL(5, "end1 "); { size_t const r = ZSTD_endStream(zc, &outBuff); @@ -391,8 +396,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo outBuff.size = ZSTD_compressBound(inSize); outBuff.pos = 0; DISPLAYLEVEL(5, "compress2 "); - { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff); - if (ZSTD_isError(r)) goto _output_error; } + CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ DISPLAYLEVEL(5, "end2 "); { size_t const r = ZSTD_endStream(zc, &outBuff); @@ -412,8 +416,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo inBuff.src = CNBuffer; inBuff.size = CNBufferSize; inBuff.pos = 0; - { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff); - if (ZSTD_isError(r)) goto _output_error; } + CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ { size_t const r = ZSTD_endStream(zc, &outBuff); if (r != 0) goto _output_error; } /* error, or some data not flushed */ @@ -462,8 +465,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo /* Memory restriction */ DISPLAYLEVEL(3, "test%3i : maxWindowSize < frame requirement : ", testNb++); ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize); - { size_t const r = ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1000); /* too small limit */ - if (ZSTD_isError(r)) goto _output_error; } + CHECK_Z( ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1000) ); /* too small limit */ inBuff.src = compressedBuffer; inBuff.size = cSize; inBuff.pos = 0; @@ -477,7 +479,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_usingCDict_advanced with masked dictID : ", testNb++); { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictionary.filled); ZSTD_frameParameters const fParams = { 1 /* contentSize */, 1 /* checksum */, 1 /* noDictID */}; - ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, 1 /* byReference */, ZSTD_dm_auto, cParams, customMem); + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dm_auto, cParams, customMem); size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, fParams, CNBufferSize); if (ZSTD_isError(initError)) goto _output_error; cSize = 0; @@ -487,8 +489,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo inBuff.src = CNBuffer; inBuff.size = CNBufferSize; inBuff.pos = 0; - { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff); - if (ZSTD_isError(r)) goto _output_error; } + CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ { size_t const r = ZSTD_endStream(zc, &outBuff); if (r != 0) goto _output_error; } /* error, or some data not flushed */ @@ -510,16 +511,14 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo } DISPLAYLEVEL(3, "test%3i : compress with ZSTD_CCtx_refPrefix : ", testNb++); - { size_t const refErr = ZSTD_CCtx_refPrefix(zc, dictionary.start, dictionary.filled); - if (ZSTD_isError(refErr)) goto _output_error; } + CHECK_Z( ZSTD_CCtx_refPrefix(zc, dictionary.start, dictionary.filled) ); outBuff.dst = compressedBuffer; outBuff.size = compressedBufferSize; outBuff.pos = 0; inBuff.src = CNBuffer; inBuff.size = CNBufferSize; inBuff.pos = 0; - { size_t const r = ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end); - if (ZSTD_isError(r)) goto _output_error; } + CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end) ); if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ cSize = outBuff.pos; DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100); @@ -546,23 +545,20 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo inBuff.src = CNBuffer; inBuff.size = CNBufferSize; inBuff.pos = 0; - { size_t const r = ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end); - if (ZSTD_isError(r)) goto _output_error; } + CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end) ); if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ cSize = outBuff.pos; DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100); DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should work): ", testNb++); - { size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize); - if (ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */ - DISPLAYLEVEL(3, "OK \n"); - } + CHECK_Z( ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize) ); + DISPLAYLEVEL(3, "OK \n"); /* Empty srcSize */ DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced with pledgedSrcSize=0 and dict : ", testNb++); { ZSTD_parameters params = ZSTD_getParams(5, 0, 0); params.fParams.contentSizeFlag = 1; - ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0); + CHECK_Z( ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0) ); } /* cstream advanced shall write content size = 0 */ inBuff.src = CNBuffer; inBuff.size = 0; @@ -570,7 +566,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo outBuff.dst = compressedBuffer; outBuff.size = compressedBufferSize; outBuff.pos = 0; - if (ZSTD_isError(ZSTD_compressStream(zc, &outBuff, &inBuff))) goto _output_error; + CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error; cSize = outBuff.pos; if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error; @@ -579,7 +575,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly : ", testNb++); { ZSTD_parameters params = ZSTD_getParams(5, 0, 0); params.fParams.contentSizeFlag = 1; - ZSTD_initCStream_advanced(zc, NULL, 0, params, 0); + CHECK_Z( ZSTD_initCStream_advanced(zc, NULL, 0, params, 0) ); } /* cstream advanced shall write content size = 0 */ inBuff.src = CNBuffer; inBuff.size = 0; @@ -587,7 +583,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo outBuff.dst = compressedBuffer; outBuff.size = compressedBufferSize; outBuff.pos = 0; - if (ZSTD_isError(ZSTD_compressStream(zc, &outBuff, &inBuff))) goto _output_error; + CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error; cSize = outBuff.pos; if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error; @@ -599,12 +595,30 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo outBuff.dst = compressedBuffer; outBuff.size = compressedBufferSize; outBuff.pos = 0; - if (ZSTD_isError(ZSTD_compressStream(zc, &outBuff, &inBuff))) goto _output_error; + CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error; cSize = outBuff.pos; if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error; DISPLAYLEVEL(3, "OK \n"); + /* Basic multithreading compression test */ + DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); + { ZSTD_parameters const params = ZSTD_getParams(1, 0, 0); + CHECK_Z( ZSTDMT_initCStream_advanced(mtctx, CNBuffer, dictSize, params, CNBufferSize) ); + } + outBuff.dst = (char*)(compressedBuffer); + outBuff.size = compressedBufferSize; + outBuff.pos = 0; + inBuff.src = CNBuffer; + inBuff.size = CNBufferSize; + inBuff.pos = 0; + CHECK_Z( ZSTDMT_compressStream_generic(mtctx, &outBuff, &inBuff, ZSTD_e_end) ); + if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ + { size_t const r = ZSTDMT_endStream(mtctx, &outBuff); + if (r != 0) goto _output_error; } /* error, or some data not flushed */ + DISPLAYLEVEL(3, "OK \n"); + + /* Overlen overwriting window data bug */ DISPLAYLEVEL(3, "test%3i : wildcopy doesn't overwrite potential match data : ", testNb++); { /* This test has a window size of 1024 bytes and consists of 3 blocks: @@ -619,9 +633,10 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo "\x28\xB5\x2F\xFD\x04\x00\x4C\x00\x00\x10\x61\x61\x01\x00\x00\x2A" "\x80\x05\x44\x00\x00\x08\x62\x01\x00\x00\x2A\x20\x04\x5D\x00\x00" "\x00\x03\x40\x00\x00\x64\x60\x27\xB0\xE0\x0C\x67\x62\xCE\xE0"; - ZSTD_DStream* zds = ZSTD_createDStream(); + ZSTD_DStream* const zds = ZSTD_createDStream(); + if (zds==NULL) goto _output_error; - ZSTD_initDStream(zds); + CHECK_Z( ZSTD_initDStream(zds) ); inBuff.src = testCase; inBuff.size = 47; inBuff.pos = 0; @@ -630,9 +645,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo outBuff.pos = 0; while (inBuff.pos < inBuff.size) { - size_t const r = ZSTD_decompressStream(zds, &outBuff, &inBuff); - /* Bug will cause checksum to fail */ - if (ZSTD_isError(r)) goto _output_error; + CHECK_Z( ZSTD_decompressStream(zds, &outBuff, &inBuff) ); } ZSTD_freeDStream(zds); @@ -643,6 +656,7 @@ _end: FUZ_freeDictionary(dictionary); ZSTD_freeCStream(zc); ZSTD_freeDStream(zd); + ZSTDMT_freeCCtx(mtctx); free(CNBuffer); free(compressedBuffer); free(decodedBuffer); @@ -687,6 +701,13 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog) #define MIN(a,b) ( (a) < (b) ? (a) : (b) ) +/* Return value in range minVal <= v <= maxVal */ +static U32 FUZ_randomClampedLength(U32* seed, U32 minVal, U32 maxVal) +{ + U32 const mod = maxVal < minVal ? 1 : (maxVal + 1) - minVal; + return (U32)((FUZ_rand(seed) % mod) + minVal); +} + #define CHECK(cond, ...) { \ if (cond) { \ DISPLAY("Error => "); \ @@ -695,35 +716,26 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog) goto _output_error; \ } } -#define CHECK_Z(f) { \ - size_t const err = f; \ - if (ZSTD_isError(err)) { \ - DISPLAY("Error => %s : %s ", \ - #f, ZSTD_getErrorName(err)); \ - DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); \ - goto _output_error; \ -} } - static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility, int bigTests) { U32 const maxSrcLog = bigTests ? 24 : 22; static const U32 maxSampleLog = 19; size_t const srcBufferSize = (size_t)1<<maxSrcLog; BYTE* cNoiseBuffer[5]; - size_t const copyBufferSize= srcBufferSize + (1<<maxSampleLog); + size_t const copyBufferSize = srcBufferSize + (1<<maxSampleLog); BYTE* const copyBuffer = (BYTE*)malloc (copyBufferSize); - size_t const cBufferSize = ZSTD_compressBound(srcBufferSize); + size_t const cBufferSize = ZSTD_compressBound(srcBufferSize); BYTE* const cBuffer = (BYTE*)malloc (cBufferSize); size_t const dstBufferSize = srcBufferSize; BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize); U32 result = 0; U32 testNb = 0; U32 coreSeed = seed; - ZSTD_CStream* zc = ZSTD_createCStream(); /* will be reset sometimes */ - ZSTD_DStream* zd = ZSTD_createDStream(); /* will be reset sometimes */ + ZSTD_CStream* zc = ZSTD_createCStream(); /* will be re-created sometimes */ + ZSTD_DStream* zd = ZSTD_createDStream(); /* will be re-created sometimes */ ZSTD_DStream* const zd_noise = ZSTD_createDStream(); clock_t const startClock = clock(); - const BYTE* dict=NULL; /* can keep same dict on 2 consecutive tests */ + const BYTE* dict = NULL; /* can keep same dict on 2 consecutive tests */ size_t dictSize = 0; U32 oldTestLog = 0; U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel() : g_cLevelMax_smallTests; @@ -801,8 +813,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */ && oldTestLog /* at least one test happened */ && resetAllowed) { maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2); - if (maxTestSize >= srcBufferSize) - maxTestSize = srcBufferSize-1; + maxTestSize = MIN(maxTestSize, srcBufferSize-16); { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize; CHECK_Z( ZSTD_resetCStream(zc, pledgedSrcSize) ); } @@ -884,12 +895,18 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize); inBuff.size = inBuff.pos + readCSrcSize; - outBuff.size = inBuff.pos + dstBuffSize; + outBuff.size = outBuff.pos + dstBuffSize; decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff); - CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult)); + if (ZSTD_getErrorCode(decompressionResult) == ZSTD_error_checksum_wrong) { + DISPLAY("checksum error : \n"); + findDiff(copyBuffer, dstBuffer, totalTestSize); + } + CHECK( ZSTD_isError(decompressionResult), "decompression error : %s", + ZSTD_getErrorName(decompressionResult) ); } CHECK (decompressionResult != 0, "frame not fully decoded"); - CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size") + CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", + (U32)outBuff.pos, (U32)totalTestSize); CHECK (inBuff.pos != cSize, "compressed data should be fully read") { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize); @@ -962,15 +979,16 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp U32 result = 0; U32 testNb = 0; U32 coreSeed = seed; - ZSTDMT_CCtx* zc = ZSTDMT_createCCtx(2); /* will be reset sometimes */ + U32 nbThreads = 2; + ZSTDMT_CCtx* zc = ZSTDMT_createCCtx(nbThreads); /* will be reset sometimes */ ZSTD_DStream* zd = ZSTD_createDStream(); /* will be reset sometimes */ ZSTD_DStream* const zd_noise = ZSTD_createDStream(); clock_t const startClock = clock(); const BYTE* dict=NULL; /* can keep same dict on 2 consecutive tests */ size_t dictSize = 0; U32 oldTestLog = 0; - U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel() : g_cLevelMax_smallTests; - U32 const nbThreadsMax = bigTests ? 5 : 2; + int const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel()-1 : g_cLevelMax_smallTests; + U32 const nbThreadsMax = bigTests ? 4 : 2; /* allocations */ cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); @@ -1006,16 +1024,16 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp size_t maxTestSize; /* init */ - if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); } - else { DISPLAYUPDATE(2, "\r%6u ", testNb); } + if (testNb < nbTests) { + DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); + } else { DISPLAYUPDATE(2, "\r%6u ", testNb); } FUZ_rand(&coreSeed); lseed = coreSeed ^ prime32; /* states full reset (deliberately not synchronized) */ /* some issues can only happen when reusing states */ if ((FUZ_rand(&lseed) & 0xFF) == 131) { - U32 const nbThreadsCandidate = (FUZ_rand(&lseed) % 6) + 1; - U32 const nbThreads = MIN(nbThreadsCandidate, nbThreadsMax); + nbThreads = (FUZ_rand(&lseed) % nbThreadsMax) + 1; DISPLAYLEVEL(5, "Creating new context with %u threads \n", nbThreads); ZSTDMT_freeCCtx(zc); zc = ZSTDMT_createCCtx(nbThreads); @@ -1055,11 +1073,12 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp } else { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog; - U32 const cLevelCandidate = (FUZ_rand(&lseed) % - (ZSTD_maxCLevel() - - (MAX(testLog, dictLog) / 3))) + - 1; - U32 const cLevel = MIN(cLevelCandidate, cLevelMax); + int const cLevelCandidate = ( FUZ_rand(&lseed) + % (ZSTD_maxCLevel() - (MAX(testLog, dictLog) / 2)) ) + + 1; + int const cLevelThreadAdjusted = cLevelCandidate - (nbThreads * 2) + 2; /* reduce cLevel when multiple threads to reduce memory consumption */ + int const cLevelMin = MAX(cLevelThreadAdjusted, 1); /* no negative cLevel yet */ + int const cLevel = MIN(cLevelMin, cLevelMax); maxTestSize = FUZ_rLogLength(&lseed, testLog); oldTestLog = testLog; /* random dictionary selection */ @@ -1142,7 +1161,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize); inBuff.size = inBuff.pos + readCSrcSize; - outBuff.size = inBuff.pos + dstBuffSize; + outBuff.size = outBuff.pos + dstBuffSize; DISPLAYLEVEL(5, "ZSTD_decompressStream input %u bytes \n", (U32)readCSrcSize); decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff); CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult)); @@ -1204,9 +1223,21 @@ _output_error: goto _cleanup; } +/** If useOpaqueAPI, sets param in cctxParams. + * Otherwise, sets the param in zc. */ +static size_t setCCtxParameter(ZSTD_CCtx* zc, ZSTD_CCtx_params* cctxParams, + ZSTD_cParameter param, unsigned value, + U32 useOpaqueAPI) +{ + if (useOpaqueAPI) { + return ZSTD_CCtxParam_setParameter(cctxParams, param, value); + } else { + return ZSTD_CCtx_setParameter(zc, param, value); + } +} /* Tests for ZSTD_compress_generic() API */ -static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double compressibility, int bigTests) +static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double compressibility, int bigTests, U32 const useOpaqueAPI) { U32 const maxSrcLog = bigTests ? 24 : 22; static const U32 maxSampleLog = 19; @@ -1228,8 +1259,10 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double const BYTE* dict = NULL; /* can keep same dict on 2 consecutive tests */ size_t dictSize = 0; U32 oldTestLog = 0; - U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel() : g_cLevelMax_smallTests; - U32 const nbThreadsMax = bigTests ? 5 : 1; + U32 windowLogMalus = 0; /* can survive between 2 loops */ + U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel()-1 : g_cLevelMax_smallTests; + U32 const nbThreadsMax = bigTests ? 4 : 2; + ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams(); /* allocations */ cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); @@ -1269,6 +1302,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double else { DISPLAYUPDATE(2, "\r%6u ", testNb); } FUZ_rand(&coreSeed); lseed = coreSeed ^ prime32; + DISPLAYLEVEL(5, " *** Test %u *** \n", testNb); /* states full reset (deliberately not synchronized) */ /* some issues can only happen when reusing states */ @@ -1308,17 +1342,19 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2); if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1; { int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1; - CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_compressionLevel, compressionLevel) ); + CHECK_Z (setCCtxParameter(zc, cctxParams, ZSTD_p_compressionLevel, compressionLevel, useOpaqueAPI) ); } } else { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog; U32 const cLevelCandidate = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - - (MAX(testLog, dictLog) / 3))) + + (MAX(testLog, dictLog) / 2))) + 1; U32 const cLevel = MIN(cLevelCandidate, cLevelMax); + DISPLAYLEVEL(5, "t%u: cLevel : %u \n", testNb, cLevel); maxTestSize = FUZ_rLogLength(&lseed, testLog); + DISPLAYLEVEL(5, "t%u: maxTestSize : %u \n", testNb, (U32)maxTestSize); oldTestLog = testLog; /* random dictionary selection */ dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0; @@ -1328,51 +1364,90 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double } { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize; ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize); + static const U32 windowLogMax = 24; /* mess with compression parameters */ cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1; + cParams.windowLog = MIN(windowLogMax, cParams.windowLog); cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1; cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1; cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1; cParams.searchLength += (FUZ_rand(&lseed) & 3) - 1; - cParams.targetLength = (U32)(cParams.targetLength * (0.5 + ((double)(FUZ_rand(&lseed) & 127) / 128))); + cParams.targetLength = (U32)((cParams.targetLength + 1 ) * (0.5 + ((double)(FUZ_rand(&lseed) & 127) / 128))); cParams = ZSTD_adjustCParams(cParams, 0, 0); - if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_windowLog, cParams.windowLog) ); - if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_hashLog, cParams.hashLog) ); - if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_chainLog, cParams.chainLog) ); - if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_searchLog, cParams.searchLog) ); - if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_minMatch, cParams.searchLength) ); - if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_targetLength, cParams.targetLength) ); - - /* unconditionally set, to be sync with decoder */ - if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_refDictContent, FUZ_rand(&lseed) & 1) ); if (FUZ_rand(&lseed) & 1) { - CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) ); - if (dict && dictSize) { - /* test that compression parameters are rejected (correctly) after loading a non-NULL dictionary */ - size_t const setError = ZSTD_CCtx_setParameter(zc, ZSTD_p_windowLog, cParams.windowLog-1) ; - CHECK(!ZSTD_isError(setError), "ZSTD_CCtx_setParameter should have failed"); - } } else { - CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) ); + CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_windowLog, cParams.windowLog, useOpaqueAPI) ); + assert(cParams.windowLog >= ZSTD_WINDOWLOG_MIN); /* guaranteed by ZSTD_adjustCParams() */ + windowLogMalus = (cParams.windowLog - ZSTD_WINDOWLOG_MIN) / 5; + DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog); + } + if (FUZ_rand(&lseed) & 1) { + CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_hashLog, cParams.hashLog, useOpaqueAPI) ); + DISPLAYLEVEL(5, "t%u: hashLog : %u \n", testNb, cParams.hashLog); + } + if (FUZ_rand(&lseed) & 1) { + CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_chainLog, cParams.chainLog, useOpaqueAPI) ); + DISPLAYLEVEL(5, "t%u: chainLog : %u \n", testNb, cParams.chainLog); } + if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_searchLog, cParams.searchLog, useOpaqueAPI) ); + if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_minMatch, cParams.searchLength, useOpaqueAPI) ); + if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_targetLength, cParams.targetLength, useOpaqueAPI) ); + + /* mess with long distance matching parameters */ + if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, useOpaqueAPI) ); + if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), useOpaqueAPI) ); + if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), useOpaqueAPI) ); + if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_LDM_BUCKETSIZELOG_MAX), useOpaqueAPI) ); + if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashEveryLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN), useOpaqueAPI) ); /* mess with frame parameters */ - if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_checksumFlag, FUZ_rand(&lseed) & 1) ); - if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_dictIDFlag, FUZ_rand(&lseed) & 1) ); - if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_contentSizeFlag, FUZ_rand(&lseed) & 1) ); + if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_checksumFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) ); + if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_dictIDFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) ); + if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_contentSizeFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) ); if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) ); - DISPLAYLEVEL(5, "pledgedSrcSize : %u \n", (U32)pledgedSrcSize); + DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (U32)pledgedSrcSize); /* multi-threading parameters */ { U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1; - U32 const nbThreads = MIN(nbThreadsCandidate, nbThreadsMax); - CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_nbThreads, nbThreads) ); + U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1; + U32 const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax); + CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_nbThreads, nbThreads, useOpaqueAPI) ); + DISPLAYLEVEL(5, "t%u: nbThreads : %u \n", testNb, nbThreads); if (nbThreads > 1) { U32 const jobLog = FUZ_rand(&lseed) % (testLog+1); - CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_overlapSizeLog, FUZ_rand(&lseed) % 10) ); - CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_jobSize, (U32)FUZ_rLogLength(&lseed, jobLog)) ); - } } } } + CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_overlapSizeLog, FUZ_rand(&lseed) % 10, useOpaqueAPI) ); + CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_jobSize, (U32)FUZ_rLogLength(&lseed, jobLog), useOpaqueAPI) ); + } + } + + if (FUZ_rand(&lseed) & 1) CHECK_Z (setCCtxParameter(zc, cctxParams, ZSTD_p_forceMaxWindow, FUZ_rand(&lseed) & 1, useOpaqueAPI) ); + + /* Apply parameters */ + if (useOpaqueAPI) { + CHECK_Z (ZSTD_CCtx_setParametersUsingCCtxParams(zc, cctxParams) ); + } + + if (FUZ_rand(&lseed) & 1) { + if (FUZ_rand(&lseed) & 1) { + CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) ); + } else { + CHECK_Z( ZSTD_CCtx_loadDictionary_byReference(zc, dict, dictSize) ); + } + if (dict && dictSize) { + /* test that compression parameters are rejected (correctly) after loading a non-NULL dictionary */ + if (useOpaqueAPI) { + size_t const setError = ZSTD_CCtx_setParametersUsingCCtxParams(zc, cctxParams); + CHECK(!ZSTD_isError(setError), "ZSTD_CCtx_setParametersUsingCCtxParams should have failed"); + } else { + size_t const setError = ZSTD_CCtx_setParameter(zc, ZSTD_p_windowLog, cParams.windowLog-1); + CHECK(!ZSTD_isError(setError), "ZSTD_CCtx_setParameter should have failed"); + } + } + } else { + CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) ); + } + } } /* multi-segments compression test */ XXH64_reset(&xxhState, 0); @@ -1389,7 +1464,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double outBuff.size = outBuff.pos + dstBuffSize; CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, flush) ); - DISPLAYLEVEL(5, "compress consumed %u bytes (total : %u) \n", + DISPLAYLEVEL(6, "compress consumed %u bytes (total : %u) \n", (U32)inBuff.pos, (U32)(totalTestSize + inBuff.pos)); XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos); @@ -1404,11 +1479,11 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1); size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize); outBuff.size = outBuff.pos + adjustedDstSize; - DISPLAYLEVEL(5, "End-flush into dst buffer of size %u \n", (U32)adjustedDstSize); + DISPLAYLEVEL(6, "End-flush into dst buffer of size %u \n", (U32)adjustedDstSize); remainingToFlush = ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end); - CHECK(ZSTD_isError(remainingToFlush), - "ZSTD_compress_generic w/ ZSTD_e_end error : %s", - ZSTD_getErrorName(remainingToFlush) ); + CHECK( ZSTD_isError(remainingToFlush), + "ZSTD_compress_generic w/ ZSTD_e_end error : %s", + ZSTD_getErrorName(remainingToFlush) ); } } crcOrig = XXH64_digest(&xxhState); cSize = outBuff.pos; @@ -1431,12 +1506,12 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize); inBuff.size = inBuff.pos + readCSrcSize; - outBuff.size = inBuff.pos + dstBuffSize; - DISPLAYLEVEL(5, "ZSTD_decompressStream input %u bytes (pos:%u/%u)\n", + outBuff.size = outBuff.pos + dstBuffSize; + DISPLAYLEVEL(6, "ZSTD_decompressStream input %u bytes (pos:%u/%u)\n", (U32)readCSrcSize, (U32)inBuff.pos, (U32)cSize); decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff); CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult)); - DISPLAYLEVEL(5, "inBuff.pos = %u \n", (U32)readCSrcSize); + DISPLAYLEVEL(6, "inBuff.pos = %u \n", (U32)readCSrcSize); } CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (U32)outBuff.pos, (U32)totalTestSize); CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (U32)inBuff.pos, (U32)cSize); @@ -1479,6 +1554,7 @@ _cleanup: ZSTD_freeCCtx(zc); ZSTD_freeDStream(zd); ZSTD_freeDStream(zd_noise); + ZSTD_freeCCtxParams(cctxParams); free(cNoiseBuffer[0]); free(cNoiseBuffer[1]); free(cNoiseBuffer[2]); @@ -1494,7 +1570,6 @@ _output_error: goto _cleanup; } - /*-******************************************************* * Command line *********************************************************/ @@ -1530,6 +1605,7 @@ int main(int argc, const char** argv) e_api selected_api = simple_api; const char* const programName = argv[0]; ZSTD_customMem const customNULL = ZSTD_defaultCMem; + U32 useOpaqueAPI = 0; /* Check command line */ for(argNb=1; argNb<argc; argNb++) { @@ -1539,8 +1615,9 @@ int main(int argc, const char** argv) /* Parsing commands. Aggregated commands are allowed */ if (argument[0]=='-') { - if (!strcmp(argument, "--mt")) { selected_api=mt_api; continue; } - if (!strcmp(argument, "--newapi")) { selected_api=advanced_api; continue; } + if (!strcmp(argument, "--mt")) { selected_api=mt_api; testNb += !testNb; continue; } + if (!strcmp(argument, "--newapi")) { selected_api=advanced_api; testNb += !testNb; continue; } + if (!strcmp(argument, "--opaqueapi")) { selected_api=advanced_api; testNb += !testNb; useOpaqueAPI = 1; continue; } if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; } argument++; @@ -1654,7 +1731,7 @@ int main(int argc, const char** argv) result = fuzzerTests_MT(seed, nbTests, testNb, ((double)proba) / 100, bigTests); break; case advanced_api : - result = fuzzerTests_newAPI(seed, nbTests, testNb, ((double)proba) / 100, bigTests); + result = fuzzerTests_newAPI(seed, nbTests, testNb, ((double)proba) / 100, bigTests, useOpaqueAPI); break; default : assert(0); /* impossible */ |