summaryrefslogtreecommitdiff
path: root/tests
diff options
context:
space:
mode:
Diffstat (limited to 'tests')
-rw-r--r--tests/Makefile22
-rw-r--r--tests/decodecorpus.c60
-rw-r--r--tests/fullbench.c4
-rw-r--r--tests/fuzzer.c264
-rw-r--r--tests/paramgrill.c82
-rwxr-xr-xtests/playTests.sh168
-rwxr-xr-xtests/test-zstd-speed.py18
-rw-r--r--tests/zbufftest.c2
-rw-r--r--tests/zstreamtest.c112
9 files changed, 564 insertions, 168 deletions
diff --git a/tests/Makefile b/tests/Makefile
index 9382fe80ddc2..ea58c0fe5119 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -26,9 +26,12 @@ PYTHON ?= python3
TESTARTEFACT := versionsTest namespaceTest
-CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
+DEBUGFLAGS=-g -DZSTD_DEBUG=1
+CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
+ -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) \
+ $(DEBUGFLAG)
CFLAGS ?= -O3
-CFLAGS += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-Wstrict-prototypes -Wundef -Wformat-security
CFLAGS += $(MOREFLAGS)
@@ -65,14 +68,16 @@ FUZZERTEST ?= -T5mn
ZSTDRTTEST = --test-large-data
DECODECORPUS_TESTTIME ?= -T30
-.PHONY: default all all32 dll clean test test32 test-all namespaceTest versionsTest
+.PHONY: default all all32 allnothread dll clean test test32 test-all namespaceTest versionsTest
default: fullbench
-all: fullbench fuzzer zstreamtest paramgrill datagen zbufftest
+all: fullbench fuzzer zstreamtest paramgrill datagen zbufftest decodecorpus
all32: fullbench32 fuzzer32 zstreamtest32 zbufftest32
+allnothread: fullbench fuzzer paramgrill datagen zbufftest decodecorpus
+
dll: fuzzer-dll zstreamtest-dll zbufftest-dll
zstd:
@@ -152,6 +157,7 @@ zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zstreamtest.c
$(MAKE) -C $(ZSTDDIR) libzstd
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)
+paramgrill : DEBUGFLAG =
paramgrill : $(ZSTD_FILES) $(PRGDIR)/datagen.c paramgrill.c
$(CC) $(FLAGS) $^ -lm -o $@$(EXT)
@@ -300,10 +306,10 @@ test-fullbench32: fullbench32 datagen
$(QEMU_SYS) ./fullbench32 -i1 -P0
test-fuzzer: fuzzer
- $(QEMU_SYS) ./fuzzer $(FUZZERTEST)
+ $(QEMU_SYS) ./fuzzer $(FUZZERTEST) $(FUZZER_FLAGS)
test-fuzzer32: fuzzer32
- $(QEMU_SYS) ./fuzzer32 $(FUZZERTEST)
+ $(QEMU_SYS) ./fuzzer32 $(FUZZERTEST) $(FUZZER_FLAGS)
test-zbuff: zbufftest
$(QEMU_SYS) ./zbufftest $(ZSTREAM_TESTTIME)
@@ -312,10 +318,10 @@ test-zbuff32: zbufftest32
$(QEMU_SYS) ./zbufftest32 $(ZSTREAM_TESTTIME)
test-zstream: zstreamtest
- $(QEMU_SYS) ./zstreamtest $(ZSTREAM_TESTTIME)
+ $(QEMU_SYS) ./zstreamtest $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
test-zstream32: zstreamtest32
- $(QEMU_SYS) ./zstreamtest32 $(ZSTREAM_TESTTIME)
+ $(QEMU_SYS) ./zstreamtest32 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
test-longmatch: longmatch
$(QEMU_SYS) ./longmatch
diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c
index 183d20f749e6..f7b3c854fdb2 100644
--- a/tests/decodecorpus.c
+++ b/tests/decodecorpus.c
@@ -98,7 +98,7 @@ static void RAND_bufferMaxSymb(U32* seed, void* ptr, size_t size, int maxSymb)
BYTE* op = ptr;
for (i = 0; i < size; i++) {
- op[i] = RAND(seed) % (maxSymb + 1);
+ op[i] = (BYTE) (RAND(seed) % (maxSymb + 1));
}
}
@@ -134,8 +134,8 @@ static void RAND_genDist(U32* seed, BYTE* dist, double weight)
{
size_t i = 0;
size_t statesLeft = DISTSIZE;
- BYTE symb = RAND(seed) % 256;
- BYTE step = (RAND(seed) % 256) | 1; /* force it to be odd so it's relatively prime to 256 */
+ BYTE symb = (BYTE) (RAND(seed) % 256);
+ BYTE step = (BYTE) ((RAND(seed) % 256) | 1); /* force it to be odd so it's relatively prime to 256 */
while (i < DISTSIZE) {
size_t states = ((size_t)(weight * statesLeft)) + 1;
@@ -259,7 +259,7 @@ static void writeFrameHeader(U32* seed, frame_t* frame)
/* Follow window algorithm from specification */
int const exponent = RAND(seed) % (MAX_WINDOW_LOG - 10);
int const mantissa = RAND(seed) % 8;
- windowByte = (exponent << 3) | mantissa;
+ windowByte = (BYTE) ((exponent << 3) | mantissa);
fh.windowSize = (1U << (exponent + 10));
fh.windowSize += fh.windowSize / 8 * mantissa;
}
@@ -284,7 +284,7 @@ static void writeFrameHeader(U32* seed, frame_t* frame)
if (contentSizeFlag && (fh.contentSize == 0 || !(RAND(seed) & 7))) {
/* do single segment sometimes */
- fh.windowSize = fh.contentSize;
+ fh.windowSize = (U32) fh.contentSize;
singleSegment = 1;
}
}
@@ -307,7 +307,7 @@ static void writeFrameHeader(U32* seed, frame_t* frame)
{
BYTE const frameHeaderDescriptor =
- (fcsCode << 6) | (singleSegment << 5) | (1 << 2);
+ (BYTE) ((fcsCode << 6) | (singleSegment << 5) | (1 << 2));
op[pos++] = frameHeaderDescriptor;
}
@@ -318,14 +318,14 @@ static void writeFrameHeader(U32* seed, frame_t* frame)
if (contentSizeFlag) {
switch (fcsCode) {
default: /* Impossible */
- case 0: op[pos++] = fh.contentSize; break;
- case 1: MEM_writeLE16(op + pos, fh.contentSize - 256); pos += 2; break;
- case 2: MEM_writeLE32(op + pos, fh.contentSize); pos += 4; break;
- case 3: MEM_writeLE64(op + pos, fh.contentSize); pos += 8; break;
+ case 0: op[pos++] = (BYTE) fh.contentSize; break;
+ case 1: MEM_writeLE16(op + pos, (U16) (fh.contentSize - 256)); pos += 2; break;
+ case 2: MEM_writeLE32(op + pos, (U32) fh.contentSize); pos += 4; break;
+ case 3: MEM_writeLE64(op + pos, (U64) fh.contentSize); pos += 8; break;
}
}
- DISPLAYLEVEL(2, " frame content size:\t%zu\n", fh.contentSize);
+ DISPLAYLEVEL(2, " frame content size:\t%u\n", (U32)fh.contentSize);
DISPLAYLEVEL(2, " frame window size:\t%u\n", fh.windowSize);
DISPLAYLEVEL(2, " content size flag:\t%d\n", contentSizeFlag);
DISPLAYLEVEL(2, " single segment flag:\t%d\n", singleSegment);
@@ -385,7 +385,7 @@ static size_t writeLiteralsBlockSimple(U32* seed, frame_t* frame, size_t content
op += litSize;
} else {
/* RLE literals */
- BYTE const symb = RAND(seed) % 256;
+ BYTE const symb = (BYTE) (RAND(seed) % 256);
DISPLAYLEVEL(4, " rle literals: 0x%02x\n", (U32)symb);
@@ -542,8 +542,8 @@ static size_t writeLiteralsBlockCompressed(U32* seed, frame_t* frame, size_t con
op += compressedSize;
compressedSize += hufHeaderSize;
- DISPLAYLEVEL(5, " regenerated size: %zu\n", litSize);
- DISPLAYLEVEL(5, " compressed size: %zu\n", compressedSize);
+ DISPLAYLEVEL(5, " regenerated size: %u\n", (U32)litSize);
+ DISPLAYLEVEL(5, " compressed size: %u\n", (U32)compressedSize);
if (compressedSize >= litSize) {
DISPLAYLEVEL(5, " trying again\n");
/* if we have to try again, reset the stats so we don't accidentally
@@ -611,7 +611,7 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
size_t const remainingMatch = contentSize - literalsSize;
size_t excessMatch = 0;
U32 numSequences = 0;
-
+
U32 i;
@@ -628,7 +628,7 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
excessMatch = remainingMatch - numSequences * MIN_SEQ_LEN;
}
- DISPLAYLEVEL(5, " total match lengths: %zu\n", remainingMatch);
+ DISPLAYLEVEL(5, " total match lengths: %u\n", (U32)remainingMatch);
for (i = 0; i < numSequences; i++) {
/* Generate match and literal lengths by exponential distribution to
@@ -647,10 +647,10 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
U32 offset, offsetCode, repIndex;
/* bounds checks */
- matchLen = MIN(matchLen, excessMatch + MIN_SEQ_LEN);
- literalLen = MIN(literalLen, literalsSize);
+ matchLen = (U32) MIN(matchLen, excessMatch + MIN_SEQ_LEN);
+ literalLen = MIN(literalLen, (U32) literalsSize);
if (i == 0 && srcPtr == frame->srcStart && literalLen == 0) literalLen = 1;
- if (i + 1 == numSequences) matchLen = MIN_SEQ_LEN + excessMatch;
+ if (i + 1 == numSequences) matchLen = MIN_SEQ_LEN + (U32) excessMatch;
memcpy(srcPtr, literals, literalLen);
srcPtr += literalLen;
@@ -694,8 +694,8 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
}
DISPLAYLEVEL(6, " LL: %5u OF: %5u ML: %5u", literalLen, offset, matchLen);
- DISPLAYLEVEL(7, " srcPos: %8tu seqNb: %3u",
- (BYTE*)srcPtr - (BYTE*)frame->srcStart, i);
+ DISPLAYLEVEL(7, " srcPos: %8u seqNb: %3u",
+ (U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart), i);
DISPLAYLEVEL(6, "\n");
if (offsetCode < 3) {
DISPLAYLEVEL(7, " repeat offset: %d\n", repIndex);
@@ -711,8 +711,8 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
memcpy(srcPtr, literals, literalsSize);
srcPtr += literalsSize;
- DISPLAYLEVEL(6, " excess literals: %5zu", literalsSize);
- DISPLAYLEVEL(7, " srcPos: %8tu", (BYTE*)srcPtr - (BYTE*)frame->srcStart);
+ DISPLAYLEVEL(6, " excess literals: %5u", (U32)literalsSize);
+ DISPLAYLEVEL(7, " srcPos: %8u", (U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart));
DISPLAYLEVEL(6, "\n");
return numSequences;
@@ -957,11 +957,11 @@ static size_t writeCompressedBlock(U32* seed, frame_t* frame, size_t contentSize
literalsSize = writeLiteralsBlock(seed, frame, contentSize);
- DISPLAYLEVEL(4, " literals size: %zu\n", literalsSize);
+ DISPLAYLEVEL(4, " literals size: %u\n", (U32)literalsSize);
nbSeq = writeSequencesBlock(seed, frame, contentSize, literalsSize);
- DISPLAYLEVEL(4, " number of sequences: %zu\n", nbSeq);
+ DISPLAYLEVEL(4, " number of sequences: %u\n", (U32)nbSeq);
return (BYTE*)frame->data - blockStart;
}
@@ -977,7 +977,7 @@ static void writeBlock(U32* seed, frame_t* frame, size_t contentSize,
BYTE *op = header + 3;
DISPLAYLEVEL(3, " block:\n");
- DISPLAYLEVEL(3, " block content size: %zu\n", contentSize);
+ DISPLAYLEVEL(3, " block content size: %u\n", (U32)contentSize);
DISPLAYLEVEL(3, " last block: %s\n", lastBlock ? "yes" : "no");
if (blockTypeDesc == 0) {
@@ -1025,10 +1025,10 @@ static void writeBlock(U32* seed, frame_t* frame, size_t contentSize,
frame->src = (BYTE*)frame->src + contentSize;
DISPLAYLEVEL(3, " block type: %s\n", BLOCK_TYPES[blockType]);
- DISPLAYLEVEL(3, " block size field: %zu\n", blockSize);
+ DISPLAYLEVEL(3, " block size field: %u\n", (U32)blockSize);
- header[0] = (lastBlock | (blockType << 1) | (blockSize << 3)) & 0xff;
- MEM_writeLE16(header + 1, blockSize >> 5);
+ header[0] = (BYTE) ((lastBlock | (blockType << 1) | (blockSize << 3)) & 0xff);
+ MEM_writeLE16(header + 1, (U16) (blockSize >> 5));
frame->data = op;
}
@@ -1300,7 +1300,7 @@ static int generateCorpus(U32 seed, unsigned numFiles, const char* const path,
*********************************************************/
static U32 makeSeed(void)
{
- U32 t = time(NULL);
+ U32 t = (U32) time(NULL);
return XXH32(&t, sizeof(t), 0) % 65536;
}
diff --git a/tests/fullbench.c b/tests/fullbench.c
index 940d315a7cd4..38cf22fa7ebc 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -251,14 +251,14 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
case 13:
benchFunction = local_ZSTD_decompressContinue; benchName = "ZSTD_decompressContinue";
break;
- case 31:
+ case 31:
benchFunction = local_ZSTD_decodeLiteralsBlock; benchName = "ZSTD_decodeLiteralsBlock";
break;
case 32:
benchFunction = local_ZSTD_decodeSeqHeaders; benchName = "ZSTD_decodeSeqHeaders";
break;
#endif
- case 41:
+ case 41:
benchFunction = local_ZSTD_compressStream; benchName = "ZSTD_compressStream";
break;
case 42:
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index def7542b5f0b..a9dcf12e0702 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -28,6 +28,7 @@
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
#include "zstd.h" /* ZSTD_VERSION_STRING */
#include "zstd_errors.h" /* ZSTD_getErrorCode */
+#include "zstdmt_compress.h"
#define ZDICT_STATIC_LINKING_ONLY
#include "zdict.h" /* ZDICT_trainFromBuffer */
#include "datagen.h" /* RDG_genBuffer */
@@ -57,7 +58,7 @@ static U32 g_displayLevel = 2;
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
if ((FUZ_clockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
{ g_displayClock = clock(); DISPLAY(__VA_ARGS__); \
- if (g_displayLevel>=4) fflush(stdout); } }
+ if (g_displayLevel>=4) fflush(stderr); } }
static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6;
static clock_t g_displayClock = 0;
@@ -133,13 +134,21 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "OK : %s \n", errorString);
}
+
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, (U32)CNBuffSize);
CHECKPLUS(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(CNBuffSize),
CNBuffer, CNBuffSize, 1),
cSize=r );
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
- DISPLAYLEVEL(4, "test%3i : decompressed size test : ", testNb++);
+
+ DISPLAYLEVEL(4, "test%3i : ZSTD_getFrameContentSize test : ", testNb++);
+ { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
+ if (rSize != CNBuffSize) goto _output_error;
+ }
+ DISPLAYLEVEL(4, "OK \n");
+
+ DISPLAYLEVEL(4, "test%3i : ZSTD_findDecompressedSize test : ", testNb++);
{ unsigned long long const rSize = ZSTD_findDecompressedSize(compressedBuffer, cSize);
if (rSize != CNBuffSize) goto _output_error;
}
@@ -157,6 +166,7 @@ static int basicUnitTests(U32 seed, double compressibility)
} }
DISPLAYLEVEL(4, "OK \n");
+
DISPLAYLEVEL(4, "test%3i : decompress with null dict : ", testNb++);
{ size_t const r = ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL, 0);
if (r != CNBuffSize) goto _output_error; }
@@ -179,6 +189,49 @@ static int basicUnitTests(U32 seed, double compressibility)
if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
+
+ /* ZSTDMT simple MT compression test */
+ DISPLAYLEVEL(4, "test%3i : create ZSTDMT CCtx : ", testNb++);
+ { ZSTDMT_CCtx* mtctx = ZSTDMT_createCCtx(2);
+ if (mtctx==NULL) {
+ DISPLAY("mtctx : mot enough memory, aborting \n");
+ testResult = 1;
+ goto _end;
+ }
+ DISPLAYLEVEL(4, "OK \n");
+
+ DISPLAYLEVEL(4, "test%3i : compress %u bytes with 2 threads : ", testNb++, (U32)CNBuffSize);
+ CHECKPLUS(r, ZSTDMT_compressCCtx(mtctx,
+ compressedBuffer, ZSTD_compressBound(CNBuffSize),
+ CNBuffer, CNBuffSize,
+ 1),
+ cSize=r );
+ DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
+
+ DISPLAYLEVEL(4, "test%3i : decompressed size test : ", testNb++);
+ { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
+ if (rSize != CNBuffSize) {
+ DISPLAY("ZSTD_getFrameContentSize incorrect : %u != %u \n", (U32)rSize, (U32)CNBuffSize);
+ goto _output_error;
+ } }
+ DISPLAYLEVEL(4, "OK \n");
+
+ DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, (U32)CNBuffSize);
+ { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
+ if (r != CNBuffSize) goto _output_error; }
+ DISPLAYLEVEL(4, "OK \n");
+
+ DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
+ { size_t u;
+ for (u=0; u<CNBuffSize; u++) {
+ if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;;
+ } }
+ DISPLAYLEVEL(4, "OK \n");
+
+ ZSTDMT_freeCCtx(mtctx);
+ }
+
+
/* Simple API multiframe test */
DISPLAYLEVEL(4, "test%3i : compress multiple frames : ", testNb++);
{ size_t off = 0;
@@ -351,7 +404,58 @@ static int basicUnitTests(U32 seed, double compressibility)
if (r != CNBuffSize) goto _output_error);
DISPLAYLEVEL(4, "OK \n");
- DISPLAYLEVEL(4, "test%3i : compress without dictID : ", testNb++);
+ DISPLAYLEVEL(4, "test%3i : compress with preprocessed dictionary : ", testNb++);
+ { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
+ ZSTD_customMem customMem = { NULL, NULL, NULL };
+ ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 1, cParams, customMem);
+ cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
+ CNBuffer, CNBuffSize, cdict);
+ ZSTD_freeCDict(cdict);
+ if (ZSTD_isError(cSize)) goto _output_error;
+ }
+ DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
+
+ DISPLAYLEVEL(4, "test%3i : retrieve dictID from frame : ", testNb++);
+ { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
+ if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
+ }
+ DISPLAYLEVEL(4, "OK \n");
+
+ DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
+ CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
+ decodedBuffer, CNBuffSize,
+ compressedBuffer, cSize,
+ dictBuffer, dictSize),
+ if (r != CNBuffSize) goto _output_error);
+ DISPLAYLEVEL(4, "OK \n");
+
+ DISPLAYLEVEL(4, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++);
+ { ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ };
+ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
+ ZSTD_customMem const customMem = { NULL, NULL, NULL };
+ ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 1, cParams, customMem);
+ cSize = ZSTD_compress_usingCDict_advanced(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
+ CNBuffer, CNBuffSize, cdict, fParams);
+ ZSTD_freeCDict(cdict);
+ if (ZSTD_isError(cSize)) goto _output_error;
+ }
+ DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
+
+ DISPLAYLEVEL(4, "test%3i : try retrieving contentSize from frame : ", testNb++);
+ { U64 const contentSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
+ if (contentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
+ }
+ DISPLAYLEVEL(4, "OK (unknown)\n");
+
+ DISPLAYLEVEL(4, "test%3i : frame built without dictID should be decompressible : ", testNb++);
+ CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
+ decodedBuffer, CNBuffSize,
+ compressedBuffer, cSize,
+ dictBuffer, dictSize),
+ if (r != CNBuffSize) goto _output_error);
+ DISPLAYLEVEL(4, "OK \n");
+
+ DISPLAYLEVEL(4, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb++);
{ ZSTD_parameters p = ZSTD_getParams(3, CNBuffSize, dictSize);
p.fParams.noDictIDFlag = 1;
cSize = ZSTD_compress_advanced(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
@@ -438,9 +542,9 @@ static int basicUnitTests(U32 seed, double compressibility)
/* Decompression defense tests */
DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++);
- { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3);
+ { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3); /* too small input */
if (!ZSTD_isError(r)) goto _output_error;
- if (r != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error; }
+ if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++);
@@ -449,6 +553,24 @@ static int basicUnitTests(U32 seed, double compressibility)
if (!ZSTD_isError(r)) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
+ /* content size verification test */
+ DISPLAYLEVEL(4, "test%3i : Content size verification : ", testNb++);
+ { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+ size_t const srcSize = 5000;
+ size_t const wrongSrcSize = (srcSize + 1000);
+ ZSTD_parameters params = ZSTD_getParams(1, wrongSrcSize, 0);
+ params.fParams.contentSizeFlag = 1;
+ { size_t const result = ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize);
+ if (ZSTD_isError(result)) goto _output_error;
+ }
+ { size_t const result = ZSTD_compressEnd(cctx, decodedBuffer, CNBuffSize, CNBuffer, srcSize);
+ if (!ZSTD_isError(result)) goto _output_error;
+ if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error;
+ DISPLAYLEVEL(4, "OK : %s \n", ZSTD_getErrorName(result));
+ }
+ ZSTD_freeCCtx(cctx);
+ }
+
/* block API tests */
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
static const size_t dictSize = 65 KB;
@@ -600,6 +722,14 @@ static size_t findDiff(const void* buf1, const void* buf2, size_t max)
}
+static ZSTD_parameters FUZ_makeParams(ZSTD_compressionParameters cParams, ZSTD_frameParameters fParams)
+{
+ ZSTD_parameters params;
+ params.cParams = cParams;
+ params.fParams = fParams;
+ return params;
+}
+
static size_t FUZ_rLogLength(U32* seed, U32 logLength)
{
size_t const lengthMask = ((size_t)1 << logLength) - 1;
@@ -616,7 +746,7 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog)
#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
-static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxDurationS, double compressibility)
+static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxDurationS, double compressibility, int bigTests)
{
static const U32 maxSrcLog = 23;
static const U32 maxSampleLog = 22;
@@ -636,6 +766,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
U32 coreSeed = seed, lseed = 0;
clock_t const startClock = clock();
clock_t const maxClockSpan = maxDurationS * CLOCKS_PER_SEC;
+ int const cLevelLimiter = bigTests ? 3 : 2;
/* allocation */
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
@@ -662,7 +793,6 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
for ( ; (testNb <= nbTests) || (FUZ_clockSpan(startClock) < maxClockSpan); testNb++ ) {
size_t sampleSize, maxTestSize, totalTestSize;
size_t cSize, totalCSize, totalGenSize;
- XXH64_state_t xxhState;
U64 crcOrig;
BYTE* sampleBuffer;
const BYTE* dict;
@@ -701,7 +831,10 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
crcOrig = XXH64(sampleBuffer, sampleSize, 0);
/* compression tests */
- { unsigned const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize)/3))) + 1;
+ { unsigned const cLevel =
+ ( FUZ_rand(&lseed) %
+ (ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize) / cLevelLimiter)) )
+ + 1;
cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel);
CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed : %s", ZSTD_getErrorName(cSize));
@@ -801,7 +934,10 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
{ U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
- int const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (MAX(testLog, dictLog)/3))) + 1;
+ int const cLevel = (FUZ_rand(&lseed) %
+ (ZSTD_maxCLevel() -
+ (MAX(testLog, dictLog) / cLevelLimiter))) +
+ 1;
maxTestSize = FUZ_rLogLength(&lseed, testLog);
if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;
@@ -813,22 +949,22 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode));
} else {
ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize);
- ZSTD_frameParameters const fpar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */,
+ ZSTD_frameParameters const fPar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */,
!(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/,
0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */
- ZSTD_parameters p;
- size_t errorCode;
- p.cParams = cPar; p.fParams = fpar;
- errorCode = ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0);
+ ZSTD_parameters const p = FUZ_makeParams(cPar, fPar);
+ size_t const errorCode = ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0);
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_advanced error : %s", ZSTD_getErrorName(errorCode));
}
{ size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx, 0);
CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode));
} }
- XXH64_reset(&xxhState, 0);
ZSTD_setCCtxParameter(ctx, ZSTD_p_forceWindow, FUZ_rand(&lseed) & 1);
+
{ U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2;
U32 n;
+ XXH64_state_t xxhState;
+ XXH64_reset(&xxhState, 0);
for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) {
size_t const segmentSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const segmentStart = FUZ_rand(&lseed) % (srcBufferSize - segmentSize);
@@ -843,13 +979,14 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize);
memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize);
totalTestSize += segmentSize;
- } }
+ }
- { size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0);
- CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult));
- cSize += flushResult;
+ { size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0);
+ CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult));
+ cSize += flushResult;
+ }
+ crcOrig = XXH64_digest(&xxhState);
}
- crcOrig = XXH64_digest(&xxhState);
/* streaming decompression test */
if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */
@@ -899,7 +1036,7 @@ _output_error:
/*_*******************************************************
* Command line
*********************************************************/
-int FUZ_usage(const char* programName)
+static int FUZ_usage(const char* programName)
{
DISPLAY( "Usage :\n");
DISPLAY( " %s [args]\n", programName);
@@ -915,19 +1052,39 @@ int FUZ_usage(const char* programName)
return 0;
}
+/*! readU32FromChar() :
+ @return : unsigned integer value read from input in `char` format
+ allows and interprets K, KB, KiB, M, MB and MiB suffix.
+ Will also modify `*stringPtr`, advancing it to position where it stopped reading.
+ Note : function result can overflow if digit string > MAX_UINT */
+static unsigned readU32FromChar(const char** stringPtr)
+{
+ unsigned result = 0;
+ while ((**stringPtr >='0') && (**stringPtr <='9'))
+ result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
+ if ((**stringPtr=='K') || (**stringPtr=='M')) {
+ result <<= 10;
+ if (**stringPtr=='M') result <<= 10;
+ (*stringPtr)++ ;
+ if (**stringPtr=='i') (*stringPtr)++;
+ if (**stringPtr=='B') (*stringPtr)++;
+ }
+ return result;
+}
int main(int argc, const char** argv)
{
- U32 seed=0;
- int seedset=0;
+ U32 seed = 0;
+ int seedset = 0;
int argNb;
int nbTests = nbTestsDefault;
int testNb = 0;
U32 proba = FUZ_compressibility_default;
- int result=0;
+ int result = 0;
U32 mainPause = 0;
U32 maxDuration = 0;
- const char* programName = argv[0];
+ int bigTests = 1;
+ const char* const programName = argv[0];
/* Check command line */
for (argNb=1; argNb<argc; argNb++) {
@@ -936,81 +1093,64 @@ int main(int argc, const char** argv)
/* Handle commands. Aggregated commands are allowed */
if (argument[0]=='-') {
+
+ if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }
+
argument++;
while (*argument!=0) {
switch(*argument)
{
case 'h':
return FUZ_usage(programName);
+
case 'v':
argument++;
- g_displayLevel=4;
+ g_displayLevel = 4;
break;
+
case 'q':
argument++;
g_displayLevel--;
break;
+
case 'p': /* pause at the end */
argument++;
mainPause = 1;
break;
case 'i':
- argument++; maxDuration=0;
- nbTests=0;
- while ((*argument>='0') && (*argument<='9')) {
- nbTests *= 10;
- nbTests += *argument - '0';
- argument++;
- }
+ argument++; maxDuration = 0;
+ nbTests = readU32FromChar(&argument);
break;
case 'T':
argument++;
- nbTests=0; maxDuration=0;
- while ((*argument>='0') && (*argument<='9')) {
- maxDuration *= 10;
- maxDuration += *argument - '0';
- argument++;
- }
- if (*argument=='m') maxDuration *=60, argument++;
+ nbTests = 0;
+ maxDuration = readU32FromChar(&argument);
+ if (*argument=='s') argument++; /* seconds */
+ if (*argument=='m') maxDuration *= 60, argument++; /* minutes */
if (*argument=='n') argument++;
break;
case 's':
argument++;
- seed=0;
- seedset=1;
- while ((*argument>='0') && (*argument<='9')) {
- seed *= 10;
- seed += *argument - '0';
- argument++;
- }
+ seedset = 1;
+ seed = readU32FromChar(&argument);
break;
case 't':
argument++;
- testNb=0;
- while ((*argument>='0') && (*argument<='9')) {
- testNb *= 10;
- testNb += *argument - '0';
- argument++;
- }
+ testNb = readU32FromChar(&argument);
break;
case 'P': /* compressibility % */
argument++;
- proba=0;
- while ((*argument>='0') && (*argument<='9')) {
- proba *= 10;
- proba += *argument - '0';
- argument++;
- }
- if (proba>100) proba=100;
+ proba = readU32FromChar(&argument);
+ if (proba>100) proba = 100;
break;
default:
- return FUZ_usage(programName);
+ return (FUZ_usage(programName), 1);
} } } } /* for (argNb=1; argNb<argc; argNb++) */
/* Get Seed */
@@ -1030,7 +1170,7 @@ int main(int argc, const char** argv)
if (testNb==0)
result = basicUnitTests(0, ((double)proba) / 100); /* constant seed for predictability */
if (!result)
- result = fuzzerTests(seed, nbTests, testNb, maxDuration, ((double)proba) / 100);
+ result = fuzzerTests(seed, nbTests, testNb, maxDuration, ((double)proba) / 100, bigTests);
if (mainPause) {
int unused;
DISPLAY("Press Enter \n");
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 5eabcba2b60b..1913b54d0a54 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -58,6 +58,11 @@ static const int g_maxNbVariations = 64;
**************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
+#undef MIN
+#undef MAX
+#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
+#define MAX(a,b) ( (a) > (b) ? (a) : (b) )
+
/*-************************************
* Benchmark Parameters
@@ -106,7 +111,11 @@ static size_t BMK_findMaxMem(U64 requiredMem)
}
-# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+static U32 FUZ_rotl32(U32 x, U32 r)
+{
+ return ((x << r) | (x >> (32 - r)));
+}
+
U32 FUZ_rand(U32* src)
{
const U32 prime1 = 2654435761U;
@@ -125,7 +134,7 @@ U32 FUZ_rand(U32* src)
*********************************************************/
typedef struct {
size_t cSize;
- double cSpeed;
+ double cSpeed; /* bytes / sec */
double dSpeed;
} BMK_result_t;
@@ -141,8 +150,6 @@ typedef struct
} blockParam_t;
-#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
-
static size_t BMK_benchParam(BMK_result_t* resultPtr,
const void* srcBuffer, size_t srcSize,
ZSTD_CCtx* ctx,
@@ -165,6 +172,11 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
char name[30] = { 0 };
U64 crcOrig;
+ /* init result for early exit */
+ resultPtr->cSize = srcSize;
+ resultPtr->cSpeed = 0.;
+ resultPtr->dSpeed = 0.;
+
/* Memory allocation & restrictions */
snprintf(name, 30, "Sw%02uc%02uh%02us%02ul%1ut%03uS%1u", Wlog, Clog, Hlog, Slog, Slength, Tlength, strat);
if (!compressedBuffer || !resultBuffer || !blockTable) {
@@ -206,7 +218,6 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
size_t cSize = 0;
double fastestC = 100000000., fastestD = 100000000.;
double ratio = 0.;
- U64 crcCheck = 0;
clock_t const benchStart = clock();
DISPLAY("\r%79s\r", "");
@@ -242,8 +253,8 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
cSize = 0;
for (blockNb=0; blockNb<nbBlocks; blockNb++)
cSize += blockTable[blockNb].cSize;
- if ((double)roundClock < fastestC * CLOCKS_PER_SEC * nbLoops) fastestC = ((double)roundClock / CLOCKS_PER_SEC) / nbLoops;
ratio = (double)srcSize / (double)cSize;
+ if ((double)roundClock < fastestC * CLOCKS_PER_SEC * nbLoops) fastestC = ((double)roundClock / CLOCKS_PER_SEC) / nbLoops;
DISPLAY("\r");
DISPLAY("%1u-%s : %9u ->", loopNb, name, (U32)srcSize);
DISPLAY(" %9u (%4.3f),%7.1f MB/s", (U32)cSize, ratio, (double)srcSize / fastestC / 1000000.);
@@ -273,18 +284,18 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
resultPtr->dSpeed = (double)srcSize / fastestD;
/* CRC Checking */
- crcCheck = XXH64(resultBuffer, srcSize, 0);
- if (crcOrig!=crcCheck) {
- unsigned u;
- unsigned eBlockSize = (unsigned)(MIN(65536*2, blockSize));
- DISPLAY("\n!!! WARNING !!! Invalid Checksum : %x != %x\n", (unsigned)crcOrig, (unsigned)crcCheck);
- for (u=0; u<srcSize; u++) {
- if (((const BYTE*)srcBuffer)[u] != ((BYTE*)resultBuffer)[u]) {
- printf("Decoding error at pos %u (block %u, pos %u) \n", u, u / eBlockSize, u % eBlockSize);
- break;
- } }
- break;
- }
+ { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
+ if (crcOrig!=crcCheck) {
+ unsigned u;
+ unsigned eBlockSize = (unsigned)(MIN(65536*2, blockSize));
+ DISPLAY("\n!!! WARNING !!! Invalid Checksum : %x != %x\n", (unsigned)crcOrig, (unsigned)crcCheck);
+ for (u=0; u<srcSize; u++) {
+ if (((const BYTE*)srcBuffer)[u] != ((BYTE*)resultBuffer)[u]) {
+ printf("Decoding error at pos %u (block %u, pos %u) \n", u, u / eBlockSize, u % eBlockSize);
+ break;
+ } }
+ break;
+ } }
#endif
} }
@@ -505,8 +516,6 @@ static BYTE g_alreadyTested[PARAMTABLESIZE] = {0}; /* init to zero */
g_alreadyTested[(XXH64(sanitizeParams(p), sizeof(p), 0) >> 3) & PARAMTABLEMASK]
-#define MAX(a,b) ( (a) > (b) ? (a) : (b) )
-
static void playAround(FILE* f, winnerInfo_t* winners,
ZSTD_compressionParameters params,
const void* srcBuffer, size_t srcSize,
@@ -711,6 +720,14 @@ int benchFiles(const char** fileNamesTable, int nbFiles)
}
+static void BMK_translateAdvancedParams(ZSTD_compressionParameters params)
+{
+ DISPLAY("--zstd=windowLog=%u,chainLog=%u,hashLog=%u,searchLog=%u,searchLength=%u,targetLength=%u,strategy=%u \n",
+ params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy));
+}
+
+/* optimizeForSize():
+ * targetSpeed : expressed in MB/s */
int optimizeForSize(const char* inFileName, U32 targetSpeed)
{
FILE* const inFile = fopen( inFileName, "rb" );
@@ -723,8 +740,11 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed)
/* Memory allocation & restrictions */
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
- if (benchedSize < inFileSize)
- DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
+ if (benchedSize < inFileSize) {
+ DISPLAY("Not enough memory for '%s' \n", inFileName);
+ fclose(inFile);
+ return 11;
+ }
/* Alloc */
origBuff = malloc(benchedSize);
@@ -747,10 +767,9 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed)
/* bench */
DISPLAY("\r%79s\r", "");
DISPLAY("optimizing for %s - limit speed %u MB/s \n", inFileName, targetSpeed);
- targetSpeed *= 1000;
+ targetSpeed *= 1000000;
{ ZSTD_CCtx* const ctx = ZSTD_createCCtx();
- ZSTD_compressionParameters params;
winnerInfo_t winner;
BMK_result_t candidate;
const size_t blockSize = g_blockSize ? g_blockSize : benchedSize;
@@ -764,26 +783,28 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed)
{ const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
int i;
for (i=1; i<=maxSeeds; i++) {
- params = ZSTD_getCParams(i, blockSize, 0);
- BMK_benchParam(&candidate, origBuff, benchedSize, ctx, params);
+ ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0);
+ BMK_benchParam(&candidate, origBuff, benchedSize, ctx, CParams);
if (candidate.cSpeed < targetSpeed)
break;
if ( (candidate.cSize < winner.result.cSize)
| ((candidate.cSize == winner.result.cSize) & (candidate.cSpeed > winner.result.cSpeed)) )
{
- winner.params = params;
+ winner.params = CParams;
winner.result = candidate;
BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize);
} }
}
BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize);
+ BMK_translateAdvancedParams(winner.params);
/* start tests */
{ time_t const grillStart = time(NULL);
do {
- params = winner.params;
+ ZSTD_compressionParameters params = winner.params;
paramVariation(&params);
- if ((FUZ_rand(&g_rand) & 15) == 3) params = randomParams();
+ if ((FUZ_rand(&g_rand) & 31) == 3) params = randomParams(); /* totally random config to improve search space */
+ params = ZSTD_adjustCParams(params, blockSize, 0);
/* exclude faster if already played set of params */
if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(params))-1)) continue;
@@ -800,6 +821,7 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed)
winner.params = params;
winner.result = candidate;
BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize);
+ BMK_translateAdvancedParams(winner.params);
}
} while (BMK_timeSpan(grillStart) < g_grillDuration_s);
}
@@ -833,7 +855,7 @@ static int usage_advanced(void)
DISPLAY( " -T# : set level 1 speed objective \n");
DISPLAY( " -B# : cut input into blocks of size # (default : single block) \n");
DISPLAY( " -i# : iteration loops [1-9](default : %i) \n", NBLOOPS);
- DISPLAY( " -O# : find Optimized parameters for # target speed (default : 0) \n");
+ DISPLAY( " -O# : find Optimized parameters for # MB/s compression speed (default : 0) \n");
DISPLAY( " -S : Single run \n");
DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100);
return 0;
diff --git a/tests/playTests.sh b/tests/playTests.sh
index c493fed55358..021fd59fe2af 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -11,6 +11,7 @@ roundTripTest() {
local_p="$2"
else
local_c="$2"
+ local_p=""
fi
rm -f tmp1 tmp2
@@ -20,13 +21,36 @@ roundTripTest() {
$DIFF -q tmp1 tmp2
}
+fileRoundTripTest() {
+ if [ -n "$3" ]; then
+ local_c="$3"
+ local_p="$2"
+ else
+ local_c="$2"
+ local_p=""
+ fi
+
+ rm -f tmp.zstd tmp.md5.1 tmp.md5.2
+ $ECHO "fileRoundTripTest: ./datagen $1 $local_p > tmp && $ZSTD -v$local_c -c tmp | $ZSTD -d"
+ ./datagen $1 $local_p > tmp
+ cat tmp | $MD5SUM > tmp.md5.1
+ $ZSTD --ultra -v$local_c -c tmp | $ZSTD -d | $MD5SUM > tmp.md5.2
+ $DIFF -q tmp.md5.1 tmp.md5.2
+}
+
+isTerminal=false
+if [ -t 0 ] && [ -t 1 ]
+then
+ isTerminal=true
+fi
+
isWindows=false
-ECHO="echo"
+ECHO="echo -e"
INTOVOID="/dev/null"
case "$OS" in
Windows*)
isWindows=true
- ECHO="echo -e"
+ INTOVOID="NUL"
;;
esac
@@ -42,11 +66,17 @@ case "$UNAME" in
SunOS) DIFF="gdiff" ;;
esac
-
$ECHO "\nStarting playTests.sh isWindows=$isWindows ZSTD='$ZSTD'"
[ -n "$ZSTD" ] || die "ZSTD variable must be defined!"
+if [ -n "$(echo hello | $ZSTD -v -T2 2>&1 > $INTOVOID | grep 'multi-threading is disabled')" ]
+then
+ hasMT=""
+else
+ hasMT="true"
+fi
+
$ECHO "\n**** simple tests **** "
./datagen > tmp
@@ -72,6 +102,12 @@ cp tmp tmp2
$ZSTD tmp2 -fo && die "-o must be followed by filename "
$ECHO "test : implied stdout when input is stdin"
$ECHO bob | $ZSTD | $ZSTD -d
+if [ "$isTerminal" = true ]; then
+$ECHO "test : compressed data to terminal"
+$ECHO bob | $ZSTD && die "should have refused : compressed data to terminal"
+$ECHO "test : compressed data from terminal (a hang here is a test fail, zstd is wrongly waiting on data from terminal)"
+$ZSTD -d > $INTOVOID && die "should have refused : compressed data from terminal"
+fi
$ECHO "test : null-length file roundtrip"
$ECHO -n '' | $ZSTD - --stdout | $ZSTD -d --stdout
$ECHO "test : decompress file with wrong suffix (must fail)"
@@ -96,6 +132,14 @@ $ZSTD -q tmp && die "overwrite check failed!"
$ECHO "test : force overwrite"
$ZSTD -q -f tmp
$ZSTD -q --force tmp
+$ECHO "test : overwrite readonly file"
+rm -f tmpro tmpro.zst
+$ECHO foo > tmpro.zst
+$ECHO foo > tmpro
+chmod 400 tmpro.zst
+$ZSTD -q tmpro && die "should have refused to overwrite read-only file"
+$ZSTD -q -f tmpro
+rm -f tmpro tmpro.zst
$ECHO "test : file removal"
$ZSTD -f --rm tmp
ls tmp && die "tmp should no longer be present"
@@ -156,6 +200,19 @@ $ECHO "$ECHO foo | $ZSTD > /dev/full"
$ECHO foo | $ZSTD > /dev/full && die "write error not detected!"
$ECHO "$ECHO foo | $ZSTD | $ZSTD -d > /dev/full"
$ECHO foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!"
+
+$ECHO "\n**** symbolic link test **** "
+
+rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst
+$ECHO "hello world" > hello.tmp
+ln -s hello.tmp world.tmp
+$ZSTD world.tmp hello.tmp
+ls hello.tmp.zst || die "regular file should have been compressed!"
+ls world.tmp.zst && die "symbolic link should not have been compressed!"
+$ZSTD world.tmp hello.tmp -f
+ls world.tmp.zst || die "symbol link should have been compressed with --force"
+rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst
+
fi
@@ -227,12 +284,12 @@ $ECHO "- Create second (different) dictionary "
$ZSTD --train *.c ../programs/*.c ../programs/*.h -o tmpDictC
$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
$ECHO "- Create dictionary with short dictID"
-$ZSTD --train *.c ../programs/*.c --dictID 1 -o tmpDict1
+$ZSTD --train *.c ../programs/*.c --dictID=1 -o tmpDict1
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
$ECHO "- Create dictionary with wrong dictID parameter order (must fail)"
$ZSTD --train *.c ../programs/*.c --dictID -o 1 tmpDict1 && die "wrong order : --dictID must be followed by argument "
$ECHO "- Create dictionary with size limit"
-$ZSTD --train *.c ../programs/*.c -o tmpDict2 --maxdict 4K -v
+$ZSTD --train *.c ../programs/*.c -o tmpDict2 --maxdict=4K -v
$ECHO "- Create dictionary with wrong parameter order (must fail)"
$ZSTD --train *.c ../programs/*.c -o tmpDict2 --maxdict -v 4K && die "wrong order : --maxdict must be followed by argument "
$ECHO "- Compress without dictID"
@@ -240,7 +297,7 @@ $ZSTD -f tmp -D tmpDict1 --no-dictID
$ZSTD -d tmp.zst -D tmpDict -fo result
$DIFF $TESTFILE result
$ECHO "- Compress with wrong argument order (must fail)"
-$ZSTD tmp -Df tmpDict1 -c > /dev/null && die "-D must be followed by dictionary name "
+$ZSTD tmp -Df tmpDict1 -c > $INTOVOID && die "-D must be followed by dictionary name "
$ECHO "- Compress multiple files with dictionary"
rm -rf dirTestDict
mkdir dirTestDict
@@ -255,6 +312,11 @@ case "$UNAME" in
*) $MD5SUM -c tmph1 ;;
esac
rm -rf dirTestDict
+$ECHO "- dictionary builder on bogus input"
+$ECHO "Hello World" > tmp
+$ZSTD --train-legacy -q tmp && die "Dictionary training should fail : not enough input source"
+./datagen -P0 -g10M > tmp
+$ZSTD --train-legacy -q tmp && die "Dictionary training should fail : source is pure noise"
rm tmp*
@@ -263,19 +325,39 @@ $ECHO "\n**** cover dictionary tests **** "
TESTFILE=../programs/zstdcli.c
./datagen > tmpDict
$ECHO "- Create first dictionary"
-$ZSTD --train --cover=k=46,d=8 *.c ../programs/*.c -o tmpDict
+$ZSTD --train-cover=k=46,d=8 *.c ../programs/*.c -o tmpDict
cp $TESTFILE tmp
$ZSTD -f tmp -D tmpDict
$ZSTD -d tmp.zst -D tmpDict -fo result
$DIFF $TESTFILE result
$ECHO "- Create second (different) dictionary"
-$ZSTD --train --cover=k=56,d=8 *.c ../programs/*.c ../programs/*.h -o tmpDictC
+$ZSTD --train-cover=k=56,d=8 *.c ../programs/*.c ../programs/*.h -o tmpDictC
$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
$ECHO "- Create dictionary with short dictID"
-$ZSTD --train --cover=k=46,d=8 *.c ../programs/*.c --dictID 1 -o tmpDict1
+$ZSTD --train-cover=k=46,d=8 *.c ../programs/*.c --dictID=1 -o tmpDict1
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
$ECHO "- Create dictionary with size limit"
-$ZSTD --train --optimize-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict 4K
+$ZSTD --train-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
+rm tmp*
+
+$ECHO "\n**** legacy dictionary tests **** "
+
+TESTFILE=../programs/zstdcli.c
+./datagen > tmpDict
+$ECHO "- Create first dictionary"
+$ZSTD --train-legacy=selectivity=8 *.c ../programs/*.c -o tmpDict
+cp $TESTFILE tmp
+$ZSTD -f tmp -D tmpDict
+$ZSTD -d tmp.zst -D tmpDict -fo result
+$DIFF $TESTFILE result
+$ECHO "- Create second (different) dictionary"
+$ZSTD --train-legacy=s=5 *.c ../programs/*.c ../programs/*.h -o tmpDictC
+$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
+$ECHO "- Create dictionary with short dictID"
+$ZSTD --train-legacy -s5 *.c ../programs/*.c --dictID=1 -o tmpDict1
+cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
+$ECHO "- Create dictionary with size limit"
+$ZSTD --train-legacy -s9 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
rm tmp*
@@ -341,7 +423,7 @@ if [ $GZIPMODE -eq 1 ]; then
$ZSTD -f --format=gzip tmp
$ZSTD -f tmp
cat tmp.gz tmp.zst tmp.gz tmp.zst | $ZSTD -d -f -o tmp
- head -c -1 tmp.gz | $ZSTD -t && die "incomplete frame not detected !"
+ head -c -1 tmp.gz | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !"
rm tmp*
else
$ECHO "gzip mode not supported"
@@ -383,13 +465,48 @@ if [ $LZMAMODE -eq 1 ]; then
$ZSTD -f --format=lzma tmp
$ZSTD -f tmp
cat tmp.xz tmp.lzma tmp.zst tmp.lzma tmp.xz tmp.zst | $ZSTD -d -f -o tmp
- head -c -1 tmp.xz | $ZSTD -t && die "incomplete frame not detected !"
- head -c -1 tmp.lzma | $ZSTD -t && die "incomplete frame not detected !"
+ head -c -1 tmp.xz | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !"
+ head -c -1 tmp.lzma | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !"
rm tmp*
else
$ECHO "xz mode not supported"
fi
+$ECHO "\n**** lz4 compatibility tests **** "
+
+LZ4MODE=1
+$ZSTD --format=lz4 -V || LZ4MODE=0
+if [ $LZ4MODE -eq 1 ]; then
+ $ECHO "lz4 support detected"
+ LZ4EXE=1
+ lz4 -V || LZ4EXE=0
+ if [ $LZ4EXE -eq 1 ]; then
+ ./datagen > tmp
+ $ZSTD --format=lz4 -f tmp
+ lz4 -t -v tmp.lz4
+ lz4 -f tmp
+ $ZSTD -d -f -v tmp.lz4
+ rm tmp*
+ else
+ $ECHO "lz4 binary not detected"
+ fi
+else
+ $ECHO "lz4 mode not supported"
+fi
+
+
+$ECHO "\n**** lz4 frame tests **** "
+
+if [ $LZ4MODE -eq 1 ]; then
+ ./datagen > tmp
+ $ZSTD -f --format=lz4 tmp
+ $ZSTD -f tmp
+ cat tmp.lz4 tmp.zst tmp.lz4 tmp.zst | $ZSTD -d -f -o tmp
+ head -c -1 tmp.lz4 | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !"
+ rm tmp*
+else
+ $ECHO "lz4 mode not supported"
+fi
$ECHO "\n**** zstd round-trip tests **** "
@@ -402,6 +519,19 @@ roundTripTest -g519K 6 # greedy, hash chain
roundTripTest -g517K 16 # btlazy2
roundTripTest -g516K 19 # btopt
+fileRoundTripTest -g500K
+
+if [ -n "$hasMT" ]
+then
+ $ECHO "\n**** zstdmt round-trip tests **** "
+ roundTripTest -g4M "1 -T0"
+ roundTripTest -g8M "3 -T2"
+ roundTripTest -g8000K "2 --threads=2"
+ fileRoundTripTest -g4M "19 -T2 -B1M"
+else
+ $ECHO "\n**** no multithreading, skipping zstdmt tests **** "
+fi
+
rm tmp*
if [ "$1" != "--test-large-data" ]; then
@@ -437,4 +567,16 @@ roundTripTest -g50000000 -P94 19
roundTripTest -g99000000 -P99 20
roundTripTest -g6000000000 -P99 1
+fileRoundTripTest -g4193M -P99 1
+
+if [ -n "$hasMT" ]
+then
+ $ECHO "\n**** zstdmt long round-trip tests **** "
+ roundTripTest -g99000000 -P99 "20 -T2"
+ roundTripTest -g6000000000 -P99 "1 -T2"
+ fileRoundTripTest -g4193M -P98 " -T0"
+else
+ $ECHO "\n**** no multithreading, skipping zstdmt tests **** "
+fi
+
rm tmp*
diff --git a/tests/test-zstd-speed.py b/tests/test-zstd-speed.py
index 23d4f477c7f3..56108a5cae4c 100755
--- a/tests/test-zstd-speed.py
+++ b/tests/test-zstd-speed.py
@@ -14,14 +14,15 @@
# - dir1/zstd and dir2/zstd will be merged in a single results file
import argparse
-import os
+import os # getloadavg
import string
import subprocess
-import time
+import time # strftime
import traceback
import hashlib
+import platform # system
-script_version = 'v1.1.1 (2016-10-28)'
+script_version = 'v1.1.2 (2017-03-26)'
default_repo_url = 'https://github.com/facebook/zstd.git'
working_dir_name = 'speedTest'
working_path = os.getcwd() + '/' + working_dir_name # /path/to/zstd/tests/speedTest
@@ -152,10 +153,15 @@ def benchmark_and_compare(branch, commit, last_commit, args, executableName, md5
% (os.getloadavg()[0], args.maxLoadAvg, sleepTime))
time.sleep(sleepTime)
start_load = str(os.getloadavg())
+ osType = platform.system()
+ if osType == 'Linux':
+ cpuSelector = "taskset --cpu-list 0"
+ else:
+ cpuSelector = ""
if args.dictionary:
- result = execute('programs/%s -rqi5b1e%s -D %s %s' % (executableName, args.lastCLevel, args.dictionary, testFilePath), print_output=True)
+ result = execute('%s programs/%s -rqi5b1e%s -D %s %s' % (cpuSelector, executableName, args.lastCLevel, args.dictionary, testFilePath), print_output=True)
else:
- result = execute('programs/%s -rqi5b1e%s %s' % (executableName, args.lastCLevel, testFilePath), print_output=True)
+ result = execute('%s programs/%s -rqi5b1e%s %s' % (cpuSelector, executableName, args.lastCLevel, testFilePath), print_output=True)
end_load = str(os.getloadavg())
linesExpected = args.lastCLevel + 1
if len(result) != linesExpected:
@@ -291,7 +297,7 @@ if __name__ == '__main__':
log("ERROR: e-mail senders 'mail' or 'mutt' not found")
exit(1)
- clang_version = execute("clang -v 2>&1 | grep 'clang version' | sed -e 's:.*version \\([0-9.]*\\).*:\\1:' -e 's:\\.\\([0-9][0-9]\\):\\1:g'", verbose)[0];
+ clang_version = execute("clang -v 2>&1 | grep ' version ' | sed -e 's:.*version \\([0-9.]*\\).*:\\1:' -e 's:\\.\\([0-9][0-9]\\):\\1:g'", verbose)[0];
gcc_version = execute("gcc -dumpversion", verbose)[0];
if verbose:
diff --git a/tests/zbufftest.c b/tests/zbufftest.c
index 14b73923311d..601aa808d027 100644
--- a/tests/zbufftest.c
+++ b/tests/zbufftest.c
@@ -60,7 +60,7 @@ static U32 g_displayLevel = 2;
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
if ((FUZ_GetClockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
{ g_displayClock = clock(); DISPLAY(__VA_ARGS__); \
- if (g_displayLevel>=4) fflush(stdout); } }
+ if (g_displayLevel>=4) fflush(stderr); } }
static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100;
static clock_t g_displayClock = 0;
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index aa7367bcfd31..0e09e185642b 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -59,7 +59,7 @@ static U32 g_displayLevel = 2;
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
if ((FUZ_GetClockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
{ g_displayClock = clock(); DISPLAY(__VA_ARGS__); \
- if (g_displayLevel>=4) fflush(stdout); } }
+ if (g_displayLevel>=4) fflush(stderr); } }
static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6;
static clock_t g_displayClock = 0;
@@ -131,7 +131,7 @@ static buffer_t FUZ_createDictionary(const void* src, size_t srcSize, size_t blo
}
{ size_t const dictSize = ZDICT_trainFromBuffer(dict.start, requestedDictSize, src, blockSizes, (unsigned)nbBlocks);
free(blockSizes);
- if (ZDICT_isError(dictSize)) { free(dict.start); return (buffer_t){ NULL, 0, 0 }; }
+ if (ZDICT_isError(dictSize)) { free(dict.start); return g_nullBuffer; }
dict.size = requestedDictSize;
dict.filled = dictSize;
return dict; /* how to return dictSize ? */
@@ -207,6 +207,16 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s);
}
+ /* Attempt bad compression parameters */
+ DISPLAYLEVEL(3, "test%3i : use bad compression parameters : ", testNb++);
+ { size_t r;
+ ZSTD_parameters params = ZSTD_getParams(1, 0, 0);
+ params.cParams.searchLength = 2;
+ r = ZSTD_initCStream_advanced(zc, NULL, 0, params, 0);
+ if (!ZSTD_isError(r)) goto _output_error;
+ DISPLAYLEVEL(3, "init error : %s \n", ZSTD_getErrorName(r));
+ }
+
/* skippable frame test */
DISPLAYLEVEL(3, "test%3i : decompress skippable frame : ", testNb++);
ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
@@ -438,11 +448,64 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
if (!ZSTD_isError(r)) goto _output_error; /* must fail : frame requires > 100 bytes */
DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); }
- /* Unknown srcSize */
+ DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_usingCDict_advanced with masked dictID : ", testNb++);
+ { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictionary.filled);
+ ZSTD_frameParameters const fParams = { 1 /* contentSize */, 1 /* checksum */, 1 /* noDictID */};
+ ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, 1 /* byReference */, cParams, customMem);
+ size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, CNBufferSize, fParams);
+ if (ZSTD_isError(initError)) goto _output_error;
+ cSize = 0;
+ outBuff.dst = compressedBuffer;
+ outBuff.size = compressedBufferSize;
+ outBuff.pos = 0;
+ inBuff.src = CNBuffer;
+ inBuff.size = CNBufferSize;
+ inBuff.pos = 0;
+ { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
+ if (ZSTD_isError(r)) goto _output_error; }
+ if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
+ { size_t const r = ZSTD_endStream(zc, &outBuff);
+ if (r != 0) goto _output_error; } /* error, or some data not flushed */
+ cSize = outBuff.pos;
+ ZSTD_freeCDict(cdict);
+ DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);
+ }
+
+ DISPLAYLEVEL(3, "test%3i : try retrieving dictID from frame : ", testNb++);
+ { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
+ if (did != 0) goto _output_error;
+ }
+ DISPLAYLEVEL(3, "OK (not detected) \n");
+
+ DISPLAYLEVEL(3, "test%3i : decompress without dictionary : ", testNb++);
+ { size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
+ if (!ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */
+ DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r));
+ }
+
+ /* Empty srcSize */
+ DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced with pledgedSrcSize=0 and dict : ", testNb++);
+ { ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
+ params.fParams.contentSizeFlag = 1;
+ ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0);
+ } /* cstream advanced shall write content size = 0 */
+ inBuff.src = CNBuffer;
+ inBuff.size = 0;
+ inBuff.pos = 0;
+ outBuff.dst = compressedBuffer;
+ outBuff.size = compressedBufferSize;
+ outBuff.pos = 0;
+ if (ZSTD_isError(ZSTD_compressStream(zc, &outBuff, &inBuff))) goto _output_error;
+ if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
+ cSize = outBuff.pos;
+ if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
+ DISPLAYLEVEL(3, "OK \n");
+
DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly : ", testNb++);
{ ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
params.fParams.contentSizeFlag = 1;
- ZSTD_initCStream_advanced(zc, NULL, 0, params, 0); } /* cstream advanced should write the 0 size field */
+ ZSTD_initCStream_advanced(zc, NULL, 0, params, 0);
+ } /* cstream advanced shall write content size = 0 */
inBuff.src = CNBuffer;
inBuff.size = 0;
inBuff.pos = 0;
@@ -552,7 +615,7 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog)
#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
-static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
+static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility, int bigTests)
{
static const U32 maxSrcLog = 24;
static const U32 maxSampleLog = 19;
@@ -574,6 +637,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
const BYTE* dict=NULL; /* can keep same dict on 2 consecutive tests */
size_t dictSize = 0;
U32 oldTestLog = 0;
+ int const cLevelLimiter = bigTests ? 3 : 2;
/* allocations */
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
@@ -638,7 +702,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
&& oldTestLog /* at least one test happened */ && resetAllowed) {
maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
- if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1;
+ if (maxTestSize >= srcBufferSize)
+ maxTestSize = srcBufferSize-1;
{ U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize;
size_t const resetError = ZSTD_resetCStream(zc, pledgedSrcSize);
CHECK(ZSTD_isError(resetError), "ZSTD_resetCStream error : %s", ZSTD_getErrorName(resetError));
@@ -646,11 +711,14 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
} else {
U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
- U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (MAX(testLog, dictLog)/3))) + 1;
+ U32 const cLevel = ( FUZ_rand(&lseed) %
+ (ZSTD_maxCLevel() -
+ (MAX(testLog, dictLog) / cLevelLimiter)))
+ + 1;
maxTestSize = FUZ_rLogLength(&lseed, testLog);
oldTestLog = testLog;
/* random dictionary selection */
- dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
+ dictSize = ((FUZ_rand(&lseed)&1)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
{ size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
dict = srcBuffer + dictStart;
}
@@ -785,7 +853,7 @@ _output_error:
/* Multi-threading version of fuzzer Tests */
-static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
+static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double compressibility, int bigTests)
{
static const U32 maxSrcLog = 24;
static const U32 maxSampleLog = 19;
@@ -807,6 +875,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
const BYTE* dict=NULL; /* can keep same dict on 2 consecutive tests */
size_t dictSize = 0;
U32 oldTestLog = 0;
+ int const cLevelLimiter = bigTests ? 3 : 2;
/* allocations */
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
@@ -851,6 +920,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
/* some issues can only happen when reusing states */
if ((FUZ_rand(&lseed) & 0xFF) == 131) {
U32 const nbThreads = (FUZ_rand(&lseed) % 6) + 1;
+ DISPLAYLEVEL(5, "Creating new context with %u threads \n", nbThreads);
ZSTDMT_freeCCtx(zc);
zc = ZSTDMT_createCCtx(nbThreads);
resetAllowed=0;
@@ -888,7 +958,10 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
} else {
U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
- U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (MAX(testLog, dictLog)/3))) + 1;
+ U32 const cLevel = (FUZ_rand(&lseed) %
+ (ZSTD_maxCLevel() -
+ (MAX(testLog, dictLog) / cLevelLimiter))) +
+ 1;
maxTestSize = FUZ_rLogLength(&lseed, testLog);
oldTestLog = testLog;
/* random dictionary selection */
@@ -898,9 +971,12 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
}
{ U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize;
ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize);
- DISPLAYLEVEL(5, "Init with windowLog = %u \n", params.cParams.windowLog);
+ DISPLAYLEVEL(5, "Init with windowLog = %u and pledgedSrcSize = %u \n",
+ params.cParams.windowLog, (U32)pledgedSrcSize);
params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
+ params.fParams.contentSizeFlag = pledgedSrcSize>0;
+ DISPLAYLEVEL(5, "checksumFlag : %u \n", params.fParams.checksumFlag);
{ size_t const initError = ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize);
CHECK (ZSTD_isError(initError),"ZSTDMT_initCStream_advanced error : %s", ZSTD_getErrorName(initError)); }
ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapSectionLog, FUZ_rand(&lseed) % 12);
@@ -938,7 +1014,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
outBuff.size = outBuff.pos + adjustedDstSize;
DISPLAYLEVEL(5, "Flushing into dst buffer of size %u \n", (U32)adjustedDstSize);
{ size_t const flushError = ZSTDMT_flushStream(zc, &outBuff);
- CHECK (ZSTD_isError(flushError), "flush error : %s", ZSTD_getErrorName(flushError));
+ CHECK (ZSTD_isError(flushError), "ZSTDMT_flushStream error : %s", ZSTD_getErrorName(flushError));
} } }
/* final frame epilogue */
@@ -949,12 +1025,12 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
outBuff.size = outBuff.pos + adjustedDstSize;
DISPLAYLEVEL(5, "Ending into dst buffer of size %u \n", (U32)adjustedDstSize);
remainingToFlush = ZSTDMT_endStream(zc, &outBuff);
- CHECK (ZSTD_isError(remainingToFlush), "flush error : %s", ZSTD_getErrorName(remainingToFlush));
+ CHECK (ZSTD_isError(remainingToFlush), "ZSTDMT_endStream error : %s", ZSTD_getErrorName(remainingToFlush));
DISPLAYLEVEL(5, "endStream : remainingToFlush : %u \n", (U32)remainingToFlush);
} }
- DISPLAYLEVEL(5, "Frame completed \n");
crcOrig = XXH64_digest(&xxhState);
cSize = outBuff.pos;
+ DISPLAYLEVEL(5, "Frame completed : %u bytes \n", (U32)cSize);
}
/* multi - fragments decompression test */
@@ -972,8 +1048,10 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
inBuff.size = inBuff.pos + readCSrcSize;
outBuff.size = inBuff.pos + dstBuffSize;
+ DISPLAYLEVEL(5, "ZSTD_decompressStream input %u bytes \n", (U32)readCSrcSize);
decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
+ DISPLAYLEVEL(5, "inBuff.pos = %u \n", (U32)readCSrcSize);
}
CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (U32)outBuff.pos, (U32)totalTestSize);
CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (U32)inBuff.pos, (U32)cSize);
@@ -1063,6 +1141,7 @@ int main(int argc, const char** argv)
int result=0;
int mainPause = 0;
int mtOnly = 0;
+ int bigTests = 1;
const char* const programName = argv[0];
ZSTD_customMem const customMem = { allocFunction, freeFunction, NULL };
ZSTD_customMem const customNULL = { NULL, NULL, NULL };
@@ -1076,6 +1155,7 @@ int main(int argc, const char** argv)
if (argument[0]=='-') {
if (!strcmp(argument, "--mt")) { mtOnly=1; continue; }
+ if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }
argument++;
while (*argument!=0) {
@@ -1181,8 +1261,8 @@ int main(int argc, const char** argv)
result = basicUnitTests(0, ((double)proba) / 100, customMem); /* use custom memory allocation functions */
} }
- if (!result && !mtOnly) result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);
- if (!result) result = fuzzerTests_MT(seed, nbTests, testNb, ((double)proba) / 100);
+ if (!result && !mtOnly) result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, bigTests);
+ if (!result) result = fuzzerTests_MT(seed, nbTests, testNb, ((double)proba) / 100, bigTests);
if (mainPause) {
int unused;
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-31 18:45:51 +0000