summaryrefslogtreecommitdiff
path: root/tests
diff options
context:
space:
mode:
Diffstat (limited to 'tests')
-rw-r--r--tests/Makefile35
-rw-r--r--tests/decodecorpus.c20
-rw-r--r--tests/fullbench.c11
-rw-r--r--tests/fuzzer.c127
-rw-r--r--tests/paramgrill.c68
-rwxr-xr-xtests/playTests.sh111
-rw-r--r--tests/seqgen.c260
-rw-r--r--tests/seqgen.h58
-rw-r--r--tests/zbufftest.c32
-rw-r--r--tests/zstreamtest.c379
10 files changed, 850 insertions, 251 deletions
diff --git a/tests/Makefile b/tests/Makefile
index 651833bbbaaee..853f4ee89b195 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -76,16 +76,16 @@ allnothread: fullbench fuzzer paramgrill datagen decodecorpus
dll: fuzzer-dll zstreamtest-dll
zstd:
- $(MAKE) -C $(PRGDIR) $@
+ $(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)"
zstd32:
- $(MAKE) -C $(PRGDIR) $@
+ $(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)"
zstd-nolegacy:
- $(MAKE) -C $(PRGDIR) $@
+ $(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)"
gzstd:
- $(MAKE) -C $(PRGDIR) zstd HAVE_ZLIB=1
+ $(MAKE) -C $(PRGDIR) zstd HAVE_ZLIB=1 MOREFLAGS="$(DEBUGFLAGS)"
fullbench32: CPPFLAGS += -m32
fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP)
@@ -130,15 +130,12 @@ zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zbufftest.c
$(MAKE) -C $(ZSTDDIR) libzstd
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)
-ZSTREAMFILES := $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c zstreamtest.c
-zstreamtest : CPPFLAGS += $(MULTITHREAD_CPP)
-zstreamtest : LDFLAGS += $(MULTITHREAD_LD)
-zstreamtest : $(ZSTREAMFILES)
- $(CC) $(FLAGS) $^ -o $@$(EXT)
-
+ZSTREAMFILES := $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c seqgen.c zstreamtest.c
zstreamtest32 : CFLAGS += -m32
-zstreamtest32 : $(ZSTREAMFILES)
- $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
+zstreamtest zstreamtest32 : CPPFLAGS += $(MULTITHREAD_CPP)
+zstreamtest zstreamtest32 : LDFLAGS += $(MULTITHREAD_LD)
+zstreamtest zstreamtest32 : $(ZSTREAMFILES)
+ $(CC) $(FLAGS) $^ -o $@$(EXT)
zstreamtest_asan : CFLAGS += -fsanitize=address
zstreamtest_asan : $(ZSTREAMFILES)
@@ -303,10 +300,10 @@ test-fullbench32: fullbench32 datagen
$(QEMU_SYS) ./fullbench32 -i1 -P0
test-fuzzer: fuzzer
- $(QEMU_SYS) ./fuzzer $(FUZZERTEST) $(FUZZER_FLAGS)
+ $(QEMU_SYS) ./fuzzer -v $(FUZZERTEST) $(FUZZER_FLAGS)
test-fuzzer32: fuzzer32
- $(QEMU_SYS) ./fuzzer32 $(FUZZERTEST) $(FUZZER_FLAGS)
+ $(QEMU_SYS) ./fuzzer32 -v $(FUZZERTEST) $(FUZZER_FLAGS)
test-zbuff: zbufftest
$(QEMU_SYS) ./zbufftest $(ZSTREAM_TESTTIME)
@@ -315,7 +312,7 @@ test-zbuff32: zbufftest32
$(QEMU_SYS) ./zbufftest32 $(ZSTREAM_TESTTIME)
test-zstream: zstreamtest
- $(QEMU_SYS) ./zstreamtest $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
+ $(QEMU_SYS) ./zstreamtest -v $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
$(QEMU_SYS) ./zstreamtest --mt -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
$(QEMU_SYS) ./zstreamtest --newapi -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
$(QEMU_SYS) ./zstreamtest --opaqueapi -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
@@ -377,9 +374,9 @@ test-pool: poolTests
test-lz4: ZSTD = LD_LIBRARY_PATH=/usr/local/lib $(PRGDIR)/zstd
test-lz4: ZSTD_LZ4 = LD_LIBRARY_PATH=/usr/local/lib ./lz4
test-lz4: ZSTD_UNLZ4 = LD_LIBRARY_PATH=/usr/local/lib ./unlz4
-test-lz4: zstd decodecorpus
- ln -s $(PRGDIR)/zstd lz4
- ln -s $(PRGDIR)/zstd unlz4
+test-lz4: zstd decodecorpus datagen
+ [ -f lz4 ] || ln -s $(PRGDIR)/zstd lz4
+ [ -f unlz4 ] || ln -s $(PRGDIR)/zstd unlz4
./decodecorpus -ptmp
# lz4 -> zstd
@@ -405,6 +402,8 @@ test-lz4: zstd decodecorpus
$(ZSTD) -d | \
cmp - tmp
+ ./datagen -g384KB | $(ZSTD) --format=lz4 | $(ZSTD) -d > /dev/null
+
rm tmp lz4 unlz4
endif
diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c
index e697e519cfc1e..407653119dd25 100644
--- a/tests/decodecorpus.c
+++ b/tests/decodecorpus.c
@@ -14,8 +14,8 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
-#include <time.h>
+#include "util.h"
#include "zstd.h"
#include "zstd_internal.h"
#include "mem.h"
@@ -49,20 +49,16 @@ static U32 g_displayLevel = 2;
#define DISPLAYUPDATE(...) \
do { \
- if ((clockSpan(g_displayClock) > g_refreshRate) || \
+ if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || \
(g_displayLevel >= 4)) { \
- g_displayClock = clock(); \
+ g_displayClock = UTIL_getTime(); \
DISPLAY(__VA_ARGS__); \
if (g_displayLevel >= 4) fflush(stderr); \
} \
} while (0)
-static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6;
-static clock_t g_displayClock = 0;
-static clock_t clockSpan(clock_t cStart)
-{
- return clock() - cStart; /* works even when overflow; max span ~ 30mn */
-}
+static const U64 g_refreshRate = SEC_TO_MICRO / 6;
+static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
#define CHECKERR(code) \
do { \
@@ -1531,14 +1527,14 @@ static int runTestMode(U32 seed, unsigned numFiles, unsigned const testDurationS
{
unsigned fnum;
- clock_t const startClock = clock();
- clock_t const maxClockSpan = testDurationS * CLOCKS_PER_SEC;
+ UTIL_time_t const startClock = UTIL_getTime();
+ U64 const maxClockSpan = testDurationS * SEC_TO_MICRO;
if (numFiles == 0 && !testDurationS) numFiles = 1;
DISPLAY("seed: %u\n", seed);
- for (fnum = 0; fnum < numFiles || clockSpan(startClock) < maxClockSpan; fnum++) {
+ for (fnum = 0; fnum < numFiles || UTIL_clockSpanMicro(startClock) < maxClockSpan; fnum++) {
if (fnum < numFiles)
DISPLAYUPDATE("\r%u/%u ", fnum, numFiles);
else
diff --git a/tests/fullbench.c b/tests/fullbench.c
index db00ce217b0e0..2ec3ce5554e20 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -233,7 +233,7 @@ static ZSTD_CCtx* g_zcc = NULL;
size_t local_ZSTD_compressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize)
{
(void)buff2;
- ZSTD_compressBegin(g_zcc, 1);
+ ZSTD_compressBegin(g_zcc, 1 /* compressionLevel */);
return ZSTD_compressEnd(g_zcc, dst, dstCapacity, src, srcSize);
}
@@ -484,8 +484,8 @@ static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchN
/* Loop for each file */
int fileIdx;
for (fileIdx=0; fileIdx<nbFiles; fileIdx++) {
- const char* inFileName = fileNamesTable[fileIdx];
- FILE* inFile = fopen( inFileName, "rb" );
+ const char* const inFileName = fileNamesTable[fileIdx];
+ FILE* const inFile = fopen( inFileName, "rb" );
U64 inFileSize;
size_t benchedSize;
void* origBuff;
@@ -495,6 +495,11 @@ static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchN
/* Memory allocation & restrictions */
inFileSize = UTIL_getFileSize(inFileName);
+ if (inFileSize == UTIL_FILESIZE_UNKNOWN) {
+ DISPLAY( "Cannot measure size of %s\n", inFileName);
+ fclose(inFile);
+ return 11;
+ }
benchedSize = BMK_findMaxMem(inFileSize*3) / 3;
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
if (benchedSize < inFileSize)
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 76df77af9015d..141bf65484c1e 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -25,7 +25,7 @@
#include <stdlib.h> /* free */
#include <stdio.h> /* fgets, sscanf */
#include <string.h> /* strcmp */
-#include <time.h> /* clock_t */
+#include <assert.h>
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
#include "zstd.h" /* ZSTD_VERSION_STRING */
#include "zstd_errors.h" /* ZSTD_getErrorCode */
@@ -36,6 +36,7 @@
#include "mem.h"
#define XXH_STATIC_LINKING_ONLY
#include "xxhash.h" /* XXH64 */
+#include "util.h"
/*-************************************
@@ -56,25 +57,22 @@ static const U32 nbTestsDefault = 30000;
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static U32 g_displayLevel = 2;
-#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
- if ((FUZ_clockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
- { g_displayClock = clock(); DISPLAY(__VA_ARGS__); \
- if (g_displayLevel>=4) fflush(stdout); } }
-static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6;
-static clock_t g_displayClock = 0;
+static const U64 g_refreshRate = SEC_TO_MICRO / 6;
+static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
+#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
+ if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
+ { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
+ if (g_displayLevel>=4) fflush(stderr); } }
/*-*******************************************************
* Fuzzer functions
*********************************************************/
+#undef MIN
+#undef MAX
#define MIN(a,b) ((a)<(b)?(a):(b))
#define MAX(a,b) ((a)>(b)?(a):(b))
-static clock_t FUZ_clockSpan(clock_t cStart)
-{
- return clock() - cStart; /* works even when overflow; max span ~ 30mn */
-}
-
#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
static unsigned FUZ_rand(unsigned* src)
{
@@ -98,6 +96,22 @@ static unsigned FUZ_highbit32(U32 v32)
/*=============================================
+* Test macros
+=============================================*/
+#define CHECK_Z(f) { \
+ size_t const err = f; \
+ if (ZSTD_isError(err)) { \
+ DISPLAY("Error => %s : %s ", \
+ #f, ZSTD_getErrorName(err)); \
+ exit(1); \
+} }
+
+#define CHECK_V(var, fn) size_t const var = fn; if (ZSTD_isError(var)) goto _output_error
+#define CHECK(fn) { CHECK_V(err, fn); }
+#define CHECKPLUS(var, fn, more) { CHECK_V(var, fn); more; }
+
+
+/*=============================================
* Memory Tests
=============================================*/
#if defined(__APPLE__) && defined(__MACH__)
@@ -146,14 +160,6 @@ static void FUZ_displayMallocStats(mallocCounter_t count)
(U32)(count.totalMalloc >> 10));
}
-#define CHECK_Z(f) { \
- size_t const err = f; \
- if (ZSTD_isError(err)) { \
- DISPLAY("Error => %s : %s ", \
- #f, ZSTD_getErrorName(err)); \
- exit(1); \
-} }
-
static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
{
size_t const inSize = 64 MB + 16 MB + 4 MB + 1 MB + 256 KB + 64 KB; /* 85.3 MB */
@@ -259,10 +265,6 @@ static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
* Unit tests
=============================================*/
-#define CHECK_V(var, fn) size_t const var = fn; if (ZSTD_isError(var)) goto _output_error
-#define CHECK(fn) { CHECK_V(err, fn); }
-#define CHECKPLUS(var, fn, more) { CHECK_V(var, fn); more; }
-
static int basicUnitTests(U32 seed, double compressibility)
{
size_t const CNBuffSize = 5 MB;
@@ -359,6 +361,31 @@ static int basicUnitTests(U32 seed, double compressibility)
if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
+ DISPLAYLEVEL(4, "test%di : check CCtx size after compressing empty input : ", testNb++);
+ { ZSTD_CCtx* cctx = ZSTD_createCCtx();
+ size_t const r = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 19);
+ if (ZSTD_isError(r)) goto _output_error;
+ if (ZSTD_sizeof_CCtx(cctx) > (1U << 20)) goto _output_error;
+ ZSTD_freeCCtx(cctx);
+ }
+ DISPLAYLEVEL(4, "OK \n");
+
+ DISPLAYLEVEL(4, "test%di : re-use CCtx with expanding block size : ", testNb++);
+ { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+ ZSTD_parameters const params = ZSTD_getParams(1, ZSTD_CONTENTSIZE_UNKNOWN, 0);
+ assert(params.fParams.contentSizeFlag == 1); /* block size will be adapted if pledgedSrcSize is enabled */
+ CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, 1 /*pledgedSrcSize*/) );
+ CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 1) ); /* creates a block size of 1 */
+
+ CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* re-use same parameters */
+ { size_t const inSize = 2* 128 KB;
+ size_t const outSize = ZSTD_compressBound(inSize);
+ CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, outSize, CNBuffer, inSize) );
+ /* will fail if blockSize is not resized */
+ }
+ ZSTD_freeCCtx(cctx);
+ }
+ DISPLAYLEVEL(4, "OK \n");
/* Static CCtx tests */
#define STATIC_CCTX_LEVEL 3
@@ -517,7 +544,7 @@ static int basicUnitTests(U32 seed, double compressibility)
off += r;
if (i == segs/2) {
/* insert skippable frame */
- const U32 skipLen = 128 KB;
+ const U32 skipLen = 129 KB;
MEM_writeLE32((BYTE*)compressedBuffer + off, ZSTD_MAGIC_SKIPPABLE_START);
MEM_writeLE32((BYTE*)compressedBuffer + off + 4, skipLen);
off += skipLen + ZSTD_skippableHeaderSize;
@@ -830,6 +857,50 @@ static int basicUnitTests(U32 seed, double compressibility)
}
DISPLAYLEVEL(4, "OK \n");
+ DISPLAYLEVEL(4, "test%3i : Dictionary with non-default repcodes : ", testNb++);
+ { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
+ dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize,
+ CNBuffer, samplesSizes, nbSamples);
+ if (ZDICT_isError(dictSize)) goto _output_error;
+ /* Set all the repcodes to non-default */
+ {
+ BYTE* dictPtr = (BYTE*)dictBuffer;
+ BYTE* dictLimit = dictPtr + dictSize - 12;
+ /* Find the repcodes */
+ while (dictPtr < dictLimit &&
+ (MEM_readLE32(dictPtr) != 1 || MEM_readLE32(dictPtr + 4) != 4 ||
+ MEM_readLE32(dictPtr + 8) != 8)) {
+ ++dictPtr;
+ }
+ if (dictPtr >= dictLimit) goto _output_error;
+ MEM_writeLE32(dictPtr + 0, 10);
+ MEM_writeLE32(dictPtr + 4, 10);
+ MEM_writeLE32(dictPtr + 8, 10);
+ /* Set the last 8 bytes to 'x' */
+ memset((BYTE*)dictBuffer + dictSize - 8, 'x', 8);
+ }
+ /* The optimal parser checks all the repcodes.
+ * Make sure at least one is a match >= targetLength so that it is
+ * immediately chosen. This will make sure that the compressor and
+ * decompressor agree on at least one of the repcodes.
+ */
+ { size_t dSize;
+ BYTE data[1024];
+ ZSTD_compressionParameters const cParams = ZSTD_getCParams(19, CNBuffSize, dictSize);
+ ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
+ ZSTD_dlm_byRef, ZSTD_dm_auto,
+ cParams, ZSTD_defaultCMem);
+ memset(data, 'x', sizeof(data));
+ cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize,
+ data, sizeof(data), cdict);
+ ZSTD_freeCDict(cdict);
+ if (ZSTD_isError(cSize)) { DISPLAYLEVEL(5, "Compression error %s : ", ZSTD_getErrorName(cSize)); goto _output_error; }
+ dSize = ZSTD_decompress_usingDict(dctx, decodedBuffer, sizeof(data), compressedBuffer, cSize, dictBuffer, dictSize);
+ if (ZSTD_isError(dSize)) { DISPLAYLEVEL(5, "Decompression error %s : ", ZSTD_getErrorName(dSize)); goto _output_error; }
+ if (memcmp(data, decodedBuffer, sizeof(data))) { DISPLAYLEVEL(5, "Data corruption : "); goto _output_error; }
+ }
+ DISPLAYLEVEL(4, "OK \n");
+
ZSTD_freeCCtx(cctx);
free(dictBuffer);
free(samplesSizes);
@@ -1200,8 +1271,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
U32 result = 0;
U32 testNb = 0;
U32 coreSeed = seed, lseed = 0;
- clock_t const startClock = clock();
- clock_t const maxClockSpan = maxDurationS * CLOCKS_PER_SEC;
+ UTIL_time_t const startClock = UTIL_getTime();
+ U64 const maxClockSpan = maxDurationS * SEC_TO_MICRO;
int const cLevelLimiter = bigTests ? 3 : 2;
/* allocation */
@@ -1226,7 +1297,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed);
/* main test loop */
- for ( ; (testNb <= nbTests) || (FUZ_clockSpan(startClock) < maxClockSpan); testNb++ ) {
+ for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < maxClockSpan); testNb++ ) {
size_t sampleSize, maxTestSize, totalTestSize;
size_t cSize, totalCSize, totalGenSize;
U64 crcOrig;
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 317ec461ca912..ae14aa0748194 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -17,13 +17,14 @@
#include <stdio.h> /* fprintf, fopen, ftello64 */
#include <string.h> /* strcmp */
#include <math.h> /* log */
-#include <time.h> /* clock_t */
+#include <time.h>
#include "mem.h"
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_estimateCCtxSize */
#include "zstd.h"
#include "datagen.h"
#include "xxhash.h"
+#include "util.h"
/*-************************************
@@ -39,7 +40,7 @@
#define GB *(1ULL<<30)
#define NBLOOPS 2
-#define TIMELOOP (2 * CLOCKS_PER_SEC)
+#define TIMELOOP (2 * SEC_TO_MICRO)
#define NB_LEVELS_TRACKED 30
@@ -49,8 +50,8 @@ static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t
static const size_t sampleSize = 10000000;
static const double g_grillDuration_s = 90000; /* about 24 hours */
-static const clock_t g_maxParamTime = 15 * CLOCKS_PER_SEC;
-static const clock_t g_maxVariationTime = 60 * CLOCKS_PER_SEC;
+static const U64 g_maxParamTime = 15 * SEC_TO_MICRO;
+static const U64 g_maxVariationTime = 60 * SEC_TO_MICRO;
static const int g_maxNbVariations = 64;
@@ -88,13 +89,9 @@ void BMK_SetNbIterations(int nbLoops)
* Private functions
*********************************************************/
-/* works even if overflow ; max span ~ 30 mn */
-static clock_t BMK_clockSpan(clock_t cStart) { return clock() - cStart; }
-
/* accuracy in seconds only, span can be multiple years */
static double BMK_timeSpan(time_t tStart) { return difftime(time(NULL), tStart); }
-
static size_t BMK_findMaxMem(U64 requiredMem)
{
size_t const step = 64 MB;
@@ -221,7 +218,7 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
size_t cSize = 0;
double fastestC = 100000000., fastestD = 100000000.;
double ratio = 0.;
- clock_t const benchStart = clock();
+ UTIL_time_t const benchStart = UTIL_getTime();
DISPLAY("\r%79s\r", "");
memset(&params, 0, sizeof(params));
@@ -229,9 +226,10 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) {
int nbLoops;
U32 blockNb;
- clock_t roundStart, roundClock;
+ UTIL_time_t roundStart;
+ U64 roundClock;
- { clock_t const benchTime = BMK_clockSpan(benchStart);
+ { U64 const benchTime = UTIL_clockSpanMicro(benchStart);
if (benchTime > g_maxParamTime) break; }
/* Compression */
@@ -239,10 +237,9 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
memset(compressedBuffer, 0xE5, maxCompressedSize);
nbLoops = 0;
- roundStart = clock();
- while (clock() == roundStart);
- roundStart = clock();
- while (BMK_clockSpan(roundStart) < TIMELOOP) {
+ UTIL_waitForNextTick();
+ roundStart = UTIL_getTime();
+ while (UTIL_clockSpanMicro(roundStart) < TIMELOOP) {
for (blockNb=0; blockNb<nbBlocks; blockNb++)
blockTable[blockNb].cSize = ZSTD_compress_advanced(ctx,
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
@@ -251,13 +248,13 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
params);
nbLoops++;
}
- roundClock = BMK_clockSpan(roundStart);
+ roundClock = UTIL_clockSpanMicro(roundStart);
cSize = 0;
for (blockNb=0; blockNb<nbBlocks; blockNb++)
cSize += blockTable[blockNb].cSize;
ratio = (double)srcSize / (double)cSize;
- if ((double)roundClock < fastestC * CLOCKS_PER_SEC * nbLoops) fastestC = ((double)roundClock / CLOCKS_PER_SEC) / nbLoops;
+ if ((double)roundClock < fastestC * SEC_TO_MICRO * nbLoops) fastestC = ((double)roundClock / SEC_TO_MICRO) / nbLoops;
DISPLAY("\r");
DISPLAY("%1u-%s : %9u ->", loopNb, name, (U32)srcSize);
DISPLAY(" %9u (%4.3f),%7.1f MB/s", (U32)cSize, ratio, (double)srcSize / fastestC / 1000000.);
@@ -269,17 +266,16 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
memset(resultBuffer, 0xD6, srcSize);
nbLoops = 0;
- roundStart = clock();
- while (clock() == roundStart);
- roundStart = clock();
- for ( ; BMK_clockSpan(roundStart) < TIMELOOP; nbLoops++) {
+ UTIL_waitForNextTick();
+ roundStart = UTIL_getTime();
+ for ( ; UTIL_clockSpanMicro(roundStart) < TIMELOOP; nbLoops++) {
for (blockNb=0; blockNb<nbBlocks; blockNb++)
blockTable[blockNb].resSize = ZSTD_decompress(blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
blockTable[blockNb].cPtr, blockTable[blockNb].cSize);
}
- roundClock = BMK_clockSpan(roundStart);
+ roundClock = UTIL_clockSpanMicro(roundStart);
- if ((double)roundClock < fastestD * CLOCKS_PER_SEC * nbLoops) fastestD = ((double)roundClock / CLOCKS_PER_SEC) / nbLoops;
+ if ((double)roundClock < fastestD * SEC_TO_MICRO * nbLoops) fastestD = ((double)roundClock / SEC_TO_MICRO) / nbLoops;
DISPLAY("\r");
DISPLAY("%1u-%s : %9u -> ", loopNb, name, (U32)srcSize);
DISPLAY("%9u (%4.3f),%7.1f MB/s, ", (U32)cSize, ratio, (double)srcSize / fastestC / 1000000.);
@@ -310,14 +306,10 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
}
-const char* g_stratName[] = { "ZSTD_fast ",
- "ZSTD_dfast ",
- "ZSTD_greedy ",
- "ZSTD_lazy ",
- "ZSTD_lazy2 ",
- "ZSTD_btlazy2 ",
- "ZSTD_btopt ",
- "ZSTD_btultra "};
+const char* g_stratName[ZSTD_btultra+1] = {
+ "(none) ", "ZSTD_fast ", "ZSTD_dfast ",
+ "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ",
+ "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "};
static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_compressionParameters params, size_t srcSize)
{
@@ -525,9 +517,9 @@ static void playAround(FILE* f, winnerInfo_t* winners,
ZSTD_CCtx* ctx)
{
int nbVariations = 0;
- clock_t const clockStart = clock();
+ UTIL_time_t const clockStart = UTIL_getTime();
- while (BMK_clockSpan(clockStart) < g_maxVariationTime) {
+ while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) {
ZSTD_compressionParameters p = params;
if (nbVariations++ > g_maxNbVariations) break;
@@ -687,6 +679,11 @@ int benchFiles(const char** fileNamesTable, int nbFiles)
DISPLAY( "Pb opening %s\n", inFileName);
return 11;
}
+ if (inFileSize == UTIL_FILESIZE_UNKNOWN) {
+ DISPLAY("Pb evaluatin size of %s \n", inFileName);
+ fclose(inFile);
+ return 11;
+ }
/* Memory allocation */
benchedSize = BMK_findMaxMem(inFileSize*3) / 3;
@@ -740,6 +737,11 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed)
/* Init */
if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; }
+ if (inFileSize == UTIL_FILESIZE_UNKNOWN) {
+ DISPLAY("Pb evaluatin size of %s \n", inFileName);
+ fclose(inFile);
+ return 11;
+ }
/* Memory allocation & restrictions */
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
diff --git a/tests/playTests.sh b/tests/playTests.sh
index bc021648cfcbe..c93c58fe8d588 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -56,10 +56,12 @@ fi
isWindows=false
INTOVOID="/dev/null"
+DEVDEVICE="/dev/zero"
case "$OS" in
Windows*)
isWindows=true
INTOVOID="NUL"
+ DEVDEVICE="NUL"
;;
esac
@@ -91,7 +93,7 @@ else
hasMT="true"
fi
-$ECHO "\n**** simple tests **** "
+$ECHO "\n===> simple tests "
./datagen > tmp
$ECHO "test : basic compression "
@@ -124,6 +126,10 @@ $ZSTD -d > $INTOVOID && die "should have refused : compressed data from terminal
fi
$ECHO "test : null-length file roundtrip"
$ECHO -n '' | $ZSTD - --stdout | $ZSTD -d --stdout
+$ECHO "test : ensure small file doesn't add 3-bytes null block"
+./datagen -g1 > tmp1
+$ZSTD tmp1 -c | wc -c | grep "14"
+$ZSTD < tmp1 | wc -c | grep "14"
$ECHO "test : decompress file with wrong suffix (must fail)"
$ZSTD -d tmpCompressed && die "wrong suffix error not detected!"
$ZSTD -df tmp && die "should have refused : wrong extension"
@@ -159,14 +165,36 @@ $ZSTD -f --rm tmp
test ! -f tmp # tmp should no longer be present
$ZSTD -f -d --rm tmp.zst
test ! -f tmp.zst # tmp.zst should no longer be present
+$ECHO "test : should quietly not remove non-regular file"
+$ECHO hello > tmp
+$ZSTD tmp -f -o "$DEVDEVICE" 2>tmplog > "$INTOVOID"
+grep -v "Refusing to remove non-regular file" tmplog
+rm -f tmplog
+$ZSTD tmp -f -o "$INTONULL" 2>&1 | grep -v "Refusing to remove non-regular file"
$ECHO "test : --rm on stdin"
$ECHO a | $ZSTD --rm > $INTOVOID # --rm should remain silent
rm tmp
$ZSTD -f tmp && die "tmp not present : should have failed"
test ! -f tmp.zst # tmp.zst should not be created
+$ECHO "test : compress multiple files"
+$ECHO hello > tmp1
+$ECHO world > tmp2
+$ZSTD tmp1 tmp2 -o "$INTOVOID"
+$ZSTD tmp1 tmp2 -c | $ZSTD -t
+$ZSTD tmp1 tmp2 -o tmp.zst
+test ! -f tmp1.zst
+test ! -f tmp2.zst
+$ZSTD tmp1 tmp2
+$ZSTD -t tmp1.zst tmp2.zst
+$ZSTD -dc tmp1.zst tmp2.zst
+$ZSTD tmp1.zst tmp2.zst -o "$INTOVOID"
+$ZSTD -d tmp1.zst tmp2.zst -o tmp
+rm tmp*
+
+
-$ECHO "\n**** Advanced compression parameters **** "
+$ECHO "\n===> Advanced compression parameters "
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!"
$ECHO "Hello world!" | $ZSTD --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!"
$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!"
@@ -180,14 +208,14 @@ roundTripTest -g512K " --long --zstd=ldmhlog=20,ldmslen=64,ldmblog=1,ldmhevery=7
roundTripTest -g512K 19
-$ECHO "\n**** Pass-Through mode **** "
+$ECHO "\n===> Pass-Through mode "
$ECHO "Hello world 1!" | $ZSTD -df
$ECHO "Hello world 2!" | $ZSTD -dcf
$ECHO "Hello world 3!" > tmp1
$ZSTD -dcf tmp1
-$ECHO "\n**** frame concatenation **** "
+$ECHO "\n===> frame concatenation "
$ECHO "hello " > hello.tmp
$ECHO "world!" > world.tmp
@@ -218,7 +246,7 @@ $ECHO "$ECHO foo | $ZSTD | $ZSTD -d > /dev/full"
$ECHO foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!"
-$ECHO "\n**** symbolic link test **** "
+$ECHO "\n===> symbolic link test "
rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst
$ECHO "hello world" > hello.tmp
@@ -233,7 +261,7 @@ rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst
fi
-$ECHO "\n**** test sparse file support **** "
+$ECHO "\n===> test sparse file support "
./datagen -g5M -P100 > tmpSparse
$ZSTD tmpSparse -c | $ZSTD -dv -o tmpSparseRegen
@@ -260,7 +288,7 @@ $DIFF tmpSparse2M tmpSparseRegenerated
rm tmpSparse*
-$ECHO "\n**** multiple files tests **** "
+$ECHO "\n===> multiple files tests "
./datagen -s1 > tmp1 2> $INTOVOID
./datagen -s2 -g100K > tmp2 2> $INTOVOID
@@ -283,7 +311,7 @@ $ECHO "compress multiple files including a missing one (notHere) : "
$ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!"
-$ECHO "\n**** dictionary tests **** "
+$ECHO "\n===> dictionary tests "
$ECHO "- test with raw dict (content only) "
./datagen > tmpDict
@@ -297,6 +325,11 @@ cp $TESTFILE tmp
$ZSTD -f tmp -D tmpDict
$ZSTD -d tmp.zst -D tmpDict -fo result
$DIFF $TESTFILE result
+if [ -n "$hasMT" ]
+then
+ $ECHO "- Test dictionary compression with multithreading "
+ ./datagen -g5M | $ZSTD -T2 -D tmpDict | $ZSTD -t -D tmpDict # fails with v1.3.2
+fi
$ECHO "- Create second (different) dictionary "
$ZSTD --train *.c ../programs/*.c ../programs/*.h -o tmpDictC
$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
@@ -339,7 +372,7 @@ $ZSTD --train-legacy -q tmp && die "Dictionary training should fail : source is
rm tmp*
-$ECHO "\n**** cover dictionary tests **** "
+$ECHO "\n===> cover dictionary builder : advanced options "
TESTFILE=../programs/zstdcli.c
./datagen > tmpDict
@@ -359,7 +392,7 @@ $ECHO "- Create dictionary with size limit"
$ZSTD --train-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
rm tmp*
-$ECHO "\n**** legacy dictionary tests **** "
+$ECHO "\n===> legacy dictionary builder "
TESTFILE=../programs/zstdcli.c
./datagen > tmpDict
@@ -380,7 +413,7 @@ $ZSTD --train-legacy -s9 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
rm tmp*
-$ECHO "\n**** integrity tests **** "
+$ECHO "\n===> integrity tests "
$ECHO "test one file (tmp1.zst) "
./datagen > tmp1
@@ -405,13 +438,13 @@ $ZSTD -t tmpSplit.* && die "bad file not detected !"
-$ECHO "\n**** golden files tests **** "
+$ECHO "\n===> golden files tests "
$ZSTD -t -r files
$ZSTD -c -r files | $ZSTD -t
-$ECHO "\n**** benchmark mode tests **** "
+$ECHO "\n===> benchmark mode tests "
$ECHO "bench one file"
./datagen > tmp1
@@ -422,7 +455,7 @@ $ECHO "with recursive and quiet modes"
$ZSTD -rqi1b1e2 tmp1
-$ECHO "\n**** gzip compatibility tests **** "
+$ECHO "\n===> gzip compatibility tests "
GZIPMODE=1
$ZSTD --format=gzip -V || GZIPMODE=0
@@ -445,7 +478,7 @@ else
fi
-$ECHO "\n**** gzip frame tests **** "
+$ECHO "\n===> gzip frame tests "
if [ $GZIPMODE -eq 1 ]; then
./datagen > tmp
@@ -459,7 +492,7 @@ else
fi
-$ECHO "\n**** xz compatibility tests **** "
+$ECHO "\n===> xz compatibility tests "
LZMAMODE=1
$ZSTD --format=xz -V || LZMAMODE=0
@@ -505,7 +538,7 @@ else
fi
-$ECHO "\n**** xz frame tests **** "
+$ECHO "\n===> xz frame tests "
if [ $LZMAMODE -eq 1 ]; then
./datagen > tmp
@@ -520,7 +553,7 @@ else
$ECHO "xz mode not supported"
fi
-$ECHO "\n**** lz4 compatibility tests **** "
+$ECHO "\n===> lz4 compatibility tests "
LZ4MODE=1
$ZSTD --format=lz4 -V || LZ4MODE=0
@@ -543,7 +576,7 @@ else
fi
-$ECHO "\n**** lz4 frame tests **** "
+$ECHO "\n===> lz4 frame tests "
if [ $LZ4MODE -eq 1 ]; then
./datagen > tmp
@@ -556,7 +589,7 @@ else
$ECHO "lz4 mode not supported"
fi
-$ECHO "\n**** zstd round-trip tests **** "
+$ECHO "\n===> zstd round-trip tests "
roundTripTest
roundTripTest -g15K # TableID==3
@@ -569,7 +602,7 @@ roundTripTest -g516K 19 # btopt
fileRoundTripTest -g500K
-$ECHO "\n**** zstd long distance matching round-trip tests **** "
+$ECHO "\n===> zstd long distance matching round-trip tests "
roundTripTest -g0 "2 --long"
roundTripTest -g1000K "1 --long"
roundTripTest -g517K "6 --long"
@@ -580,58 +613,56 @@ fileRoundTripTest -g5M "3 --long"
if [ -n "$hasMT" ]
then
- $ECHO "\n**** zstdmt round-trip tests **** "
+ $ECHO "\n===> zstdmt round-trip tests "
roundTripTest -g4M "1 -T0"
roundTripTest -g8M "3 -T2"
roundTripTest -g8000K "2 --threads=2"
fileRoundTripTest -g4M "19 -T2 -B1M"
- $ECHO "\n**** zstdmt long distance matching round-trip tests **** "
+ $ECHO "\n===> zstdmt long distance matching round-trip tests "
roundTripTest -g8M "3 --long -T2"
else
- $ECHO "\n**** no multithreading, skipping zstdmt tests **** "
+ $ECHO "\n===> no multithreading, skipping zstdmt tests "
fi
rm tmp*
-$ECHO "\n**** zstd --list/-l single frame tests ****"
+$ECHO "\n===> zstd --list/-l single frame tests "
./datagen > tmp1
./datagen > tmp2
./datagen > tmp3
$ZSTD tmp*
$ZSTD -l *.zst
-$ZSTD -lv *.zst
+$ZSTD -lv *.zst | grep "Decompressed Size:" # check that decompressed size is present in header
$ZSTD --list *.zst
$ZSTD --list -v *.zst
-$ECHO "\n**** zstd --list/-l multiple frame tests ****"
+$ECHO "\n===> zstd --list/-l multiple frame tests "
cat tmp1.zst tmp2.zst > tmp12.zst
cat tmp12.zst tmp3.zst > tmp123.zst
$ZSTD -l *.zst
$ZSTD -lv *.zst
-$ZSTD --list *.zst
-$ZSTD --list -v *.zst
-$ECHO "\n**** zstd --list/-l error detection tests ****"
+$ECHO "\n===> zstd --list/-l error detection tests "
! $ZSTD -l tmp1 tmp1.zst
! $ZSTD --list tmp*
! $ZSTD -lv tmp1*
! $ZSTD --list -v tmp2 tmp12.zst
-$ECHO "\n**** zstd --list/-l test with null files ****"
+$ECHO "\n===> zstd --list/-l test with null files "
./datagen -g0 > tmp5
$ZSTD tmp5
$ZSTD -l tmp5.zst
! $ZSTD -l tmp5*
-$ZSTD -lv tmp5.zst
+$ZSTD -lv tmp5.zst | grep "Decompressed Size: 0.00 KB (0 B)" # check that 0 size is present in header
! $ZSTD -lv tmp5*
-$ECHO "\n**** zstd --list/-l test with no content size field ****"
-./datagen -g1MB | $ZSTD > tmp6.zst
+$ECHO "\n===> zstd --list/-l test with no content size field "
+./datagen -g513K | $ZSTD > tmp6.zst
$ZSTD -l tmp6.zst
-$ZSTD -lv tmp6.zst
+! $ZSTD -lv tmp6.zst | grep "Decompressed Size:" # must NOT be present in header
-$ECHO "\n**** zstd --list/-l test with no checksum ****"
+$ECHO "\n===> zstd --list/-l test with no checksum "
$ZSTD -f --no-check tmp1
$ZSTD -l tmp1.zst
$ZSTD -lv tmp1.zst
@@ -639,7 +670,7 @@ $ZSTD -lv tmp1.zst
rm tmp*
-$ECHO "\n**** zstd long distance matching tests **** "
+$ECHO "\n===> zstd long distance matching tests "
roundTripTest -g0 " --long"
roundTripTest -g9M "2 --long"
# Test parameter parsing
@@ -654,7 +685,7 @@ if [ "$1" != "--test-large-data" ]; then
exit 0
fi
-$ECHO "\n**** large files tests **** "
+$ECHO "\n===> large files tests "
roundTripTest -g270000000 1
roundTripTest -g250000000 2
@@ -685,7 +716,7 @@ roundTripTest -g5000000000 -P99 1
fileRoundTripTest -g4193M -P99 1
-$ECHO "\n**** zstd long, long distance matching round-trip tests **** "
+$ECHO "\n===> zstd long, long distance matching round-trip tests "
roundTripTest -g270000000 "1 --long"
roundTripTest -g130000000 -P60 "5 --long"
roundTripTest -g35000000 -P70 "8 --long"
@@ -697,7 +728,7 @@ roundTripTest -g600M -P50 "1 --long --zstd=wlog=29,clog=28"
if [ -n "$hasMT" ]
then
- $ECHO "\n**** zstdmt long round-trip tests **** "
+ $ECHO "\n===> zstdmt long round-trip tests "
roundTripTest -g80000000 -P99 "19 -T2" " "
roundTripTest -g5000000000 -P99 "1 -T2" " "
roundTripTest -g500000000 -P97 "1 -T999" " "
diff --git a/tests/seqgen.c b/tests/seqgen.c
new file mode 100644
index 0000000000000..8233ecee1cfd2
--- /dev/null
+++ b/tests/seqgen.c
@@ -0,0 +1,260 @@
+/*
+ * Copyright (c) 2017-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include "seqgen.h"
+#include "mem.h"
+#include <string.h>
+
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+
+static const size_t kMatchBytes = 128;
+
+#define SEQ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+static BYTE SEQ_randByte(U32* src)
+{
+ static const U32 prime1 = 2654435761U;
+ static const U32 prime2 = 2246822519U;
+ U32 rand32 = *src;
+ rand32 *= prime1;
+ rand32 ^= prime2;
+ rand32 = SEQ_rotl32(rand32, 13);
+ *src = rand32;
+ return (BYTE)(rand32 >> 5);
+}
+
+SEQ_stream SEQ_initStream(unsigned seed)
+{
+ SEQ_stream stream;
+ stream.state = 0;
+ XXH64_reset(&stream.xxh, 0);
+ stream.seed = seed;
+ return stream;
+}
+
+/* Generates a single guard byte, then match length + 1 of a different byte,
+ * then another guard byte.
+ */
+static size_t SEQ_gen_matchLength(SEQ_stream* stream, unsigned value,
+ SEQ_outBuffer* out)
+{
+ typedef enum {
+ ml_first_byte = 0,
+ ml_match_bytes,
+ ml_last_byte,
+ } ml_state;
+ BYTE* const ostart = (BYTE*)out->dst;
+ BYTE* const oend = ostart + out->size;
+ BYTE* op = ostart + out->pos;
+
+ switch ((ml_state)stream->state) {
+ case ml_first_byte:
+ /* Generate a single byte and pick a different byte for the match */
+ if (op >= oend) {
+ stream->bytesLeft = 1;
+ break;
+ }
+ *op = SEQ_randByte(&stream->seed) & 0xFF;
+ do {
+ stream->saved = SEQ_randByte(&stream->seed) & 0xFF;
+ } while (*op == stream->saved);
+ ++op;
+ /* State transition */
+ stream->state = ml_match_bytes;
+ stream->bytesLeft = value + 1;
+ /* fall-through */
+ case ml_match_bytes: {
+ /* Copy matchLength + 1 bytes to the output buffer */
+ size_t const setLength = MIN(stream->bytesLeft, (size_t)(oend - op));
+ if (setLength > 0) {
+ memset(op, stream->saved, setLength);
+ op += setLength;
+ stream->bytesLeft -= setLength;
+ }
+ if (stream->bytesLeft > 0)
+ break;
+ /* State transition */
+ stream->state = ml_last_byte;
+ }
+ /* fall-through */
+ case ml_last_byte:
+ /* Generate a single byte and pick a different byte for the match */
+ if (op >= oend) {
+ stream->bytesLeft = 1;
+ break;
+ }
+ do {
+ *op = SEQ_randByte(&stream->seed) & 0xFF;
+ } while (*op == stream->saved);
+ ++op;
+ /* State transition */
+ /* fall-through */
+ default:
+ stream->state = 0;
+ stream->bytesLeft = 0;
+ break;
+ }
+ XXH64_update(&stream->xxh, ostart + out->pos, (op - ostart) - out->pos);
+ out->pos = op - ostart;
+ return stream->bytesLeft;
+}
+
+/* Saves the current seed then generates kMatchBytes random bytes >= 128.
+ * Generates literal length - kMatchBytes random bytes < 128.
+ * Generates another kMatchBytes using the saved seed to generate a match.
+ * This way the match is easy to find for the compressors.
+ */
+static size_t SEQ_gen_litLength(SEQ_stream* stream, unsigned value, SEQ_outBuffer* out)
+{
+ typedef enum {
+ ll_start = 0,
+ ll_run_bytes,
+ ll_literals,
+ ll_run_match,
+ } ll_state;
+ BYTE* const ostart = (BYTE*)out->dst;
+ BYTE* const oend = ostart + out->size;
+ BYTE* op = ostart + out->pos;
+
+ switch ((ll_state)stream->state) {
+ case ll_start:
+ stream->state = ll_run_bytes;
+ stream->saved = stream->seed;
+ stream->bytesLeft = MIN(kMatchBytes, value);
+ /* fall-through */
+ case ll_run_bytes:
+ while (stream->bytesLeft > 0 && op < oend) {
+ *op++ = SEQ_randByte(&stream->seed) | 0x80;
+ --stream->bytesLeft;
+ }
+ if (stream->bytesLeft > 0)
+ break;
+ /* State transition */
+ stream->state = ll_literals;
+ stream->bytesLeft = value - MIN(kMatchBytes, value);
+ /* fall-through */
+ case ll_literals:
+ while (stream->bytesLeft > 0 && op < oend) {
+ *op++ = SEQ_randByte(&stream->seed) & 0x7F;
+ --stream->bytesLeft;
+ }
+ if (stream->bytesLeft > 0)
+ break;
+ /* State transition */
+ stream->state = ll_run_match;
+ stream->bytesLeft = MIN(kMatchBytes, value);
+ /* fall-through */
+ case ll_run_match: {
+ while (stream->bytesLeft > 0 && op < oend) {
+ *op++ = SEQ_randByte(&stream->saved) | 0x80;
+ --stream->bytesLeft;
+ }
+ if (stream->bytesLeft > 0)
+ break;
+ }
+ /* fall-through */
+ default:
+ stream->state = 0;
+ stream->bytesLeft = 0;
+ break;
+ }
+ XXH64_update(&stream->xxh, ostart + out->pos, (op - ostart) - out->pos);
+ out->pos = op - ostart;
+ return stream->bytesLeft;
+}
+
+/* Saves the current seed then generates kMatchBytes random bytes >= 128.
+ * Generates offset - kMatchBytes of zeros to get a large offset without
+ * polluting the hash tables.
+ * Generates another kMatchBytes using the saved seed to generate a with the
+ * required offset.
+ */
+static size_t SEQ_gen_offset(SEQ_stream* stream, unsigned value, SEQ_outBuffer* out)
+{
+ typedef enum {
+ of_start = 0,
+ of_run_bytes,
+ of_offset,
+ of_run_match,
+ } of_state;
+ BYTE* const ostart = (BYTE*)out->dst;
+ BYTE* const oend = ostart + out->size;
+ BYTE* op = ostart + out->pos;
+
+ switch ((of_state)stream->state) {
+ case of_start:
+ stream->state = of_run_bytes;
+ stream->saved = stream->seed;
+ stream->bytesLeft = MIN(value, kMatchBytes);
+ /* fall-through */
+ case of_run_bytes: {
+ while (stream->bytesLeft > 0 && op < oend) {
+ *op++ = SEQ_randByte(&stream->seed) | 0x80;
+ --stream->bytesLeft;
+ }
+ if (stream->bytesLeft > 0)
+ break;
+ /* State transition */
+ stream->state = of_offset;
+ stream->bytesLeft = value - MIN(value, kMatchBytes);
+ }
+ /* fall-through */
+ case of_offset: {
+ /* Copy matchLength + 1 bytes to the output buffer */
+ size_t const setLength = MIN(stream->bytesLeft, (size_t)(oend - op));
+ if (setLength > 0) {
+ memset(op, 0, setLength);
+ op += setLength;
+ stream->bytesLeft -= setLength;
+ }
+ if (stream->bytesLeft > 0)
+ break;
+ /* State transition */
+ stream->state = of_run_match;
+ stream->bytesLeft = MIN(value, kMatchBytes);
+ }
+ /* fall-through */
+ case of_run_match: {
+ while (stream->bytesLeft > 0 && op < oend) {
+ *op++ = SEQ_randByte(&stream->saved) | 0x80;
+ --stream->bytesLeft;
+ }
+ if (stream->bytesLeft > 0)
+ break;
+ }
+ /* fall-through */
+ default:
+ stream->state = 0;
+ stream->bytesLeft = 0;
+ break;
+ }
+ XXH64_update(&stream->xxh, ostart + out->pos, (op - ostart) - out->pos);
+ out->pos = op - ostart;
+ return stream->bytesLeft;
+}
+
+/* Returns the number of bytes left to generate.
+ * Must pass the same type/value until it returns 0.
+ */
+size_t SEQ_gen(SEQ_stream* stream, SEQ_gen_type type, unsigned value, SEQ_outBuffer* out)
+{
+ switch (type) {
+ case SEQ_gen_ml: return SEQ_gen_matchLength(stream, value, out);
+ case SEQ_gen_ll: return SEQ_gen_litLength(stream, value, out);
+ case SEQ_gen_of: return SEQ_gen_offset(stream, value, out);
+ case SEQ_gen_max: /* fall-through */
+ default: return 0;
+ }
+}
+
+/* Returns the xxhash of the data produced so far */
+XXH64_hash_t SEQ_digest(SEQ_stream const* stream)
+{
+ return XXH64_digest(&stream->xxh);
+}
diff --git a/tests/seqgen.h b/tests/seqgen.h
new file mode 100644
index 0000000000000..72d798846aa86
--- /dev/null
+++ b/tests/seqgen.h
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2017-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef SEQGEN_H
+#define SEQGEN_H
+
+#define XXH_STATIC_LINKING_ONLY
+
+#include "xxhash.h"
+#include <stddef.h> /* size_t */
+
+typedef enum {
+ SEQ_gen_ml = 0,
+ SEQ_gen_ll,
+ SEQ_gen_of,
+ SEQ_gen_max /* Must be the last value */
+} SEQ_gen_type;
+
+/* Internal state, do not use */
+typedef struct {
+ XXH64_state_t xxh; /* xxh state for all the data produced so far (seed=0) */
+ unsigned seed;
+ int state; /* enum to control state machine (clean=0) */
+ unsigned saved;
+ size_t bytesLeft;
+} SEQ_stream;
+
+SEQ_stream SEQ_initStream(unsigned seed);
+
+typedef struct {
+ void* dst;
+ size_t size;
+ size_t pos;
+} SEQ_outBuffer;
+
+/* Returns non-zero until the current type/value has been generated.
+ * Must pass the same type/value until it returns 0.
+ *
+ * Recommended to pick a value in the middle of the range you want, since there
+ * may be some noise that causes actual results to be slightly different.
+ * We try to be more accurate for smaller values.
+ *
+ * NOTE: Very small values don't work well (< 6).
+ */
+size_t SEQ_gen(SEQ_stream* stream, SEQ_gen_type type, unsigned value,
+ SEQ_outBuffer* out);
+
+/* Returns the xxhash of the data produced so far */
+XXH64_hash_t SEQ_digest(SEQ_stream const* stream);
+
+#endif /* SEQGEN_H */
diff --git a/tests/zbufftest.c b/tests/zbufftest.c
index e64c886a5e752..9b6f7bad6dc60 100644
--- a/tests/zbufftest.c
+++ b/tests/zbufftest.c
@@ -24,7 +24,6 @@
**************************************/
#include <stdlib.h> /* free */
#include <stdio.h> /* fgets, sscanf */
-#include <time.h> /* clock_t, clock() */
#include <string.h> /* strcmp */
#include "mem.h"
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */
@@ -34,6 +33,7 @@
#include "datagen.h" /* RDG_genBuffer */
#define XXH_STATIC_LINKING_ONLY
#include "xxhash.h" /* XXH64_* */
+#include "util.h"
/*-************************************
@@ -58,26 +58,24 @@ static const U32 prime2 = 2246822519U;
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static U32 g_displayLevel = 2;
+static const U64 g_refreshRate = SEC_TO_MICRO / 6;
+static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
+
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
- if ((FUZ_GetClockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
- { g_displayClock = clock(); DISPLAY(__VA_ARGS__); \
+ if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
+ { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
if (g_displayLevel>=4) fflush(stderr); } }
-static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100;
-static clock_t g_displayClock = 0;
-static clock_t g_clockTime = 0;
+static U64 g_clockTime = 0;
/*-*******************************************************
* Fuzzer functions
*********************************************************/
+#undef MIN
+#undef MAX
+#define MIN(a,b) ((a)<(b)?(a):(b))
#define MAX(a,b) ((a)>(b)?(a):(b))
-
-static clock_t FUZ_GetClockSpan(clock_t clockStart)
-{
- return clock() - clockStart; /* works even when overflow. Max span ~ 30 mn */
-}
-
/*! FUZ_rand() :
@return : a 27 bits random value, from a 32-bits `seed`.
`seed` is also modified */
@@ -256,8 +254,6 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog)
return FUZ_rLogLength(seed, logLength);
}
-#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
-
#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
@@ -278,7 +274,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
U32 coreSeed = seed;
ZBUFF_CCtx* zc;
ZBUFF_DCtx* zd;
- clock_t startClock = clock();
+ UTIL_time_t startClock = UTIL_getTime();
/* allocations */
zc = ZBUFF_createCCtx();
@@ -308,7 +304,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
FUZ_rand(&coreSeed);
/* test loop */
- for ( ; (testNb <= nbTests) || (FUZ_GetClockSpan(startClock) < g_clockTime) ; testNb++ ) {
+ for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
U32 lseed;
const BYTE* srcBuffer;
const BYTE* dict;
@@ -357,7 +353,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
{ ZSTD_parameters params = ZSTD_getParams(cLevel, 0, dictSize);
params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
- { size_t const initError = ZBUFF_compressInit_advanced(zc, dict, dictSize, params, 0);
+ { size_t const initError = ZBUFF_compressInit_advanced(zc, dict, dictSize, params, ZSTD_CONTENTSIZE_UNKNOWN);
CHECK (ZBUFF_isError(initError),"init error : %s", ZBUFF_getErrorName(initError));
} } }
@@ -548,7 +544,7 @@ int main(int argc, const char** argv)
}
if (*argument=='m') g_clockTime *=60, argument++;
if (*argument=='n') argument++;
- g_clockTime *= CLOCKS_PER_SEC;
+ g_clockTime *= SEC_TO_MICRO;
break;
case 's':
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 8b4c836940a63..5590c9af15f8e 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -24,7 +24,6 @@
**************************************/
#include <stdlib.h> /* free */
#include <stdio.h> /* fgets, sscanf */
-#include <time.h> /* clock_t, clock() */
#include <string.h> /* strcmp */
#include <assert.h> /* assert */
#include "mem.h"
@@ -36,6 +35,8 @@
#include "datagen.h" /* RDG_genBuffer */
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
#include "xxhash.h" /* XXH64_* */
+#include "seqgen.h"
+#include "util.h"
/*-************************************
@@ -61,26 +62,24 @@ static const U32 prime32 = 2654435761U;
if (g_displayLevel>=4) fflush(stderr); }
static U32 g_displayLevel = 2;
+static const U64 g_refreshRate = SEC_TO_MICRO / 6;
+static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
+
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
- if ((FUZ_GetClockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
- { g_displayClock = clock(); DISPLAY(__VA_ARGS__); \
- if (g_displayLevel>=4) fflush(stderr); } }
-static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6;
-static clock_t g_displayClock = 0;
+ if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
+ { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
+ if (g_displayLevel>=4) fflush(stderr); } }
-static clock_t g_clockTime = 0;
+static U64 g_clockTime = 0;
/*-*******************************************************
* Fuzzer functions
*********************************************************/
+#undef MIN
+#undef MAX
+#define MIN(a,b) ((a)<(b)?(a):(b))
#define MAX(a,b) ((a)>(b)?(a):(b))
-
-static clock_t FUZ_GetClockSpan(clock_t clockStart)
-{
- return clock() - clockStart; /* works even when overflow. Max span ~ 30 mn */
-}
-
/*! FUZ_rand() :
@return : a 27 bits random value, from a 32-bits `seed`.
`seed` is also modified */
@@ -96,15 +95,21 @@ unsigned int FUZ_rand(unsigned int* seedPtr)
return rand32 >> 5;
}
-#define CHECK_Z(f) { \
- size_t const err = f; \
- if (ZSTD_isError(err)) { \
- DISPLAY("Error => %s : %s ", \
- #f, ZSTD_getErrorName(err)); \
- DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); \
+#define CHECK(cond, ...) { \
+ if (cond) { \
+ DISPLAY("Error => "); \
+ DISPLAY(__VA_ARGS__); \
+ DISPLAY(" (seed %u, test nb %u, line %u) \n", \
+ seed, testNb, __LINE__); \
goto _output_error; \
} }
+#define CHECK_Z(f) { \
+ size_t const err = f; \
+ CHECK(ZSTD_isError(err), "%s : %s ", \
+ #f, ZSTD_getErrorName(err)); \
+}
+
/*======================================================
* Basic Unit tests
@@ -144,12 +149,69 @@ static void FUZ_freeDictionary(buffer_t dict)
free(dict.start);
}
+/* Round trips data and updates xxh with the decompressed data produced */
+static size_t SEQ_roundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
+ XXH64_state_t* xxh, void* data, size_t size,
+ ZSTD_EndDirective endOp)
+{
+ static BYTE compressed[1024];
+ static BYTE uncompressed[1024];
+
+ ZSTD_inBuffer cin = {data, size, 0};
+ size_t cret;
+
+ do {
+ ZSTD_outBuffer cout = {compressed, sizeof(compressed), 0};
+ ZSTD_inBuffer din = {compressed, 0, 0};
+ ZSTD_outBuffer dout = {uncompressed, 0, 0};
+
+ cret = ZSTD_compress_generic(cctx, &cout, &cin, endOp);
+ if (ZSTD_isError(cret))
+ return cret;
+
+ din.size = cout.pos;
+ while (din.pos < din.size || (endOp == ZSTD_e_end && cret == 0)) {
+ size_t dret;
+
+ dout.pos = 0;
+ dout.size = sizeof(uncompressed);
+ dret = ZSTD_decompressStream(dctx, &dout, &din);
+ if (ZSTD_isError(dret))
+ return dret;
+ XXH64_update(xxh, dout.dst, dout.pos);
+ if (dret == 0)
+ break;
+ }
+ } while (cin.pos < cin.size || (endOp != ZSTD_e_continue && cret != 0));
+ return 0;
+}
+
+/* Generates some data and round trips it */
+static size_t SEQ_generateRoundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
+ XXH64_state_t* xxh, SEQ_stream* seq,
+ SEQ_gen_type type, unsigned value)
+{
+ static BYTE data[1024];
+ size_t gen;
+
+ do {
+ SEQ_outBuffer sout = {data, sizeof(data), 0};
+ size_t ret;
+ gen = SEQ_gen(seq, type, value, &sout);
+
+ ret = SEQ_roundTrip(cctx, dctx, xxh, sout.dst, sout.pos, ZSTD_e_continue);
+ if (ZSTD_isError(ret))
+ return ret;
+ } while (gen != 0);
+
+ return 0;
+}
-static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem)
+static int basicUnitTests(U32 seed, double compressibility)
{
size_t const CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
void* CNBuffer = malloc(CNBufferSize);
- size_t const skippableFrameSize = 11;
+ size_t const skippableFrameSize = 200 KB;
size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
void* compressedBuffer = malloc(compressedBufferSize);
size_t const decodedBufferSize = CNBufferSize;
@@ -157,8 +219,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
size_t cSize;
int testResult = 0;
U32 testNb = 1;
- ZSTD_CStream* zc = ZSTD_createCStream_advanced(customMem);
- ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem);
+ ZSTD_CStream* zc = ZSTD_createCStream();
+ ZSTD_DStream* zd = ZSTD_createDStream();
ZSTDMT_CCtx* mtctx = ZSTDMT_createCCtx(2);
ZSTD_inBuffer inBuff, inBuff2;
@@ -183,16 +245,32 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
}
dictID = ZDICT_getDictID(dictionary.start, dictionary.filled);
+ /* Basic compression test */
+ DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
+ CHECK_Z( ZSTD_initCStream(zc, 1 /* cLevel */) );
+ outBuff.dst = (char*)(compressedBuffer);
+ outBuff.size = compressedBufferSize;
+ outBuff.pos = 0;
+ inBuff.src = CNBuffer;
+ inBuff.size = CNBufferSize;
+ inBuff.pos = 0;
+ CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
+ if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
+ { size_t const r = ZSTD_endStream(zc, &outBuff);
+ if (r != 0) goto _output_error; } /* error, or some data not flushed */
+ DISPLAYLEVEL(3, "OK (%u bytes)\n", (U32)outBuff.pos);
+
/* generate skippable frame */
MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize);
cSize = skippableFrameSize + 8;
- /* Basic compression test */
- DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
- CHECK_Z( ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1) );
+ /* Basic compression test using dict */
+ DISPLAYLEVEL(3, "test%3i : skipframe + compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
+ CHECK_Z( ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1 /* cLevel */) );
outBuff.dst = (char*)(compressedBuffer)+cSize;
- outBuff.size = compressedBufferSize;
+ assert(compressedBufferSize > cSize);
+ outBuff.size = compressedBufferSize - cSize;
outBuff.pos = 0;
inBuff.src = CNBuffer;
inBuff.size = CNBufferSize;
@@ -366,7 +444,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
/* Complex context re-use scenario */
DISPLAYLEVEL(3, "test%3i : context re-use : ", testNb++);
ZSTD_freeCStream(zc);
- zc = ZSTD_createCStream_advanced(customMem);
+ zc = ZSTD_createCStream();
if (zc==NULL) goto _output_error; /* memory allocation issue */
/* use 1 */
{ size_t const inSize = 513;
@@ -408,6 +486,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
DISPLAYLEVEL(3, "test%3i : digested dictionary : ", testNb++);
{ ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, 1 /*byRef*/ );
size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict);
+ DISPLAYLEVEL(5, "ZSTD_initCStream_usingCDict result : %u ", (U32)initError);
if (ZSTD_isError(initError)) goto _output_error;
cSize = 0;
outBuff.dst = compressedBuffer;
@@ -416,10 +495,13 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
inBuff.src = CNBuffer;
inBuff.size = CNBufferSize;
inBuff.pos = 0;
+ DISPLAYLEVEL(5, "- starting ZSTD_compressStream ");
CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
- { size_t const r = ZSTD_endStream(zc, &outBuff);
- if (r != 0) goto _output_error; } /* error, or some data not flushed */
+ { size_t const r = ZSTD_endStream(zc, &outBuff);
+ DISPLAYLEVEL(5, "- ZSTD_endStream result : %u ", (U32)r);
+ if (r != 0) goto _output_error; /* error, or some data not flushed */
+ }
cSize = outBuff.pos;
ZSTD_freeCDict(cdict);
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);
@@ -442,12 +524,12 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
{ ZSTD_DDict* const ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
size_t const initError = ZSTD_initDStream_usingDDict(zd, ddict);
if (ZSTD_isError(initError)) goto _output_error;
- inBuff.src = compressedBuffer;
- inBuff.size = cSize;
- inBuff.pos = 0;
outBuff.dst = decodedBuffer;
outBuff.size = CNBufferSize;
outBuff.pos = 0;
+ inBuff.src = compressedBuffer;
+ inBuff.size = cSize;
+ inBuff.pos = 0;
{ size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
@@ -466,12 +548,12 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
DISPLAYLEVEL(3, "test%3i : maxWindowSize < frame requirement : ", testNb++);
ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
CHECK_Z( ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1000) ); /* too small limit */
- inBuff.src = compressedBuffer;
- inBuff.size = cSize;
- inBuff.pos = 0;
outBuff.dst = decodedBuffer;
outBuff.size = CNBufferSize;
outBuff.pos = 0;
+ inBuff.src = compressedBuffer;
+ inBuff.size = cSize;
+ inBuff.pos = 0;
{ size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (!ZSTD_isError(r)) goto _output_error; /* must fail : frame requires > 100 bytes */
DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); }
@@ -479,7 +561,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_usingCDict_advanced with masked dictID : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictionary.filled);
ZSTD_frameParameters const fParams = { 1 /* contentSize */, 1 /* checksum */, 1 /* noDictID */};
- ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dm_auto, cParams, customMem);
+ ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dm_auto, cParams, ZSTD_defaultCMem);
size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, fParams, CNBufferSize);
if (ZSTD_isError(initError)) goto _output_error;
cSize = 0;
@@ -558,14 +640,14 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced with pledgedSrcSize=0 and dict : ", testNb++);
{ ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
params.fParams.contentSizeFlag = 1;
- CHECK_Z( ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0) );
+ CHECK_Z( ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0 /* pledgedSrcSize==0 means "empty" when params.fParams.contentSizeFlag is set */) );
} /* cstream advanced shall write content size = 0 */
- inBuff.src = CNBuffer;
- inBuff.size = 0;
- inBuff.pos = 0;
outBuff.dst = compressedBuffer;
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
+ inBuff.src = CNBuffer;
+ inBuff.size = 0;
+ inBuff.pos = 0;
CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
cSize = outBuff.pos;
@@ -589,12 +671,12 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
ZSTD_resetCStream(zc, 0); /* resetCStream should treat 0 as unknown */
- inBuff.src = CNBuffer;
- inBuff.size = 0;
- inBuff.pos = 0;
outBuff.dst = compressedBuffer;
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
+ inBuff.src = CNBuffer;
+ inBuff.size = 0;
+ inBuff.pos = 0;
CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
cSize = outBuff.pos;
@@ -606,7 +688,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
{ ZSTD_parameters const params = ZSTD_getParams(1, 0, 0);
CHECK_Z( ZSTDMT_initCStream_advanced(mtctx, CNBuffer, dictSize, params, CNBufferSize) );
}
- outBuff.dst = (char*)(compressedBuffer);
+ outBuff.dst = compressedBuffer;
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
inBuff.src = CNBuffer;
@@ -618,6 +700,108 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
if (r != 0) goto _output_error; } /* error, or some data not flushed */
DISPLAYLEVEL(3, "OK \n");
+ /* Complex multithreading + dictionary test */
+ { U32 const nbThreads = 2;
+ size_t const jobSize = 4 * 1 MB;
+ size_t const srcSize = jobSize * nbThreads; /* we want each job to have predictable size */
+ size_t const segLength = 2 KB;
+ size_t const offset = 600 KB; /* must be larger than window defined in cdict */
+ size_t const start = jobSize + (offset-1);
+ const BYTE* const srcToCopy = (const BYTE*)CNBuffer + start;
+ BYTE* const dst = (BYTE*)CNBuffer + start - offset;
+ DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads + dictionary : ", testNb++, (U32)srcSize);
+ CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_compressionLevel, 3) );
+ CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_nbThreads, 2) );
+ CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_jobSize, jobSize) );
+ assert(start > offset);
+ assert(start + segLength < COMPRESSIBLE_NOISE_LENGTH);
+ memcpy(dst, srcToCopy, segLength); /* create a long repetition at long distance for job 2 */
+ outBuff.dst = compressedBuffer;
+ outBuff.size = compressedBufferSize;
+ outBuff.pos = 0;
+ inBuff.src = CNBuffer;
+ inBuff.size = srcSize; assert(srcSize < COMPRESSIBLE_NOISE_LENGTH);
+ inBuff.pos = 0;
+ }
+ { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 4 KB, dictionary.filled); /* intentionnally lies on estimatedSrcSize, to push cdict into targeting a small window size */
+ ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dm_fullDict, cParams, ZSTD_defaultCMem);
+ DISPLAYLEVEL(5, "cParams.windowLog = %u : ", cParams.windowLog);
+ CHECK_Z( ZSTD_CCtx_refCDict(zc, cdict) );
+ CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end) );
+ CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) ); /* do not keep a reference to cdict, as its lifetime ends */
+ ZSTD_freeCDict(cdict);
+ }
+ if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
+ cSize = outBuff.pos;
+ DISPLAYLEVEL(3, "OK \n");
+
+ DISPLAYLEVEL(3, "test%3i : decompress large frame created from multiple threads + dictionary : ", testNb++);
+ { ZSTD_DStream* const dstream = ZSTD_createDCtx();
+ ZSTD_frameHeader zfh;
+ ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize);
+ DISPLAYLEVEL(5, "frame windowsize = %u : ", (U32)zfh.windowSize);
+ outBuff.dst = decodedBuffer;
+ outBuff.size = CNBufferSize;
+ outBuff.pos = 0;
+ inBuff.src = compressedBuffer;
+ inBuff.pos = 0;
+ CHECK_Z( ZSTD_initDStream_usingDict(dstream, dictionary.start, dictionary.filled) );
+ inBuff.size = 1; /* avoid shortcut to single-pass mode */
+ CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) );
+ inBuff.size = cSize;
+ CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) );
+ if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
+ ZSTD_freeDStream(dstream);
+ }
+ DISPLAYLEVEL(3, "OK \n");
+
+ DISPLAYLEVEL(3, "test%3i : check dictionary FSE tables can represent every code : ", testNb++);
+ { unsigned const kMaxWindowLog = 24;
+ unsigned value;
+ ZSTD_compressionParameters cParams = ZSTD_getCParams(3, 1U << kMaxWindowLog, 1024);
+ ZSTD_CDict* cdict;
+ ZSTD_DDict* ddict;
+ SEQ_stream seq = SEQ_initStream(0x87654321);
+ SEQ_gen_type type;
+ XXH64_state_t xxh;
+
+ XXH64_reset(&xxh, 0);
+ cParams.windowLog = kMaxWindowLog;
+ cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dm_fullDict, cParams, ZSTD_defaultCMem);
+ ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
+
+ if (!cdict || !ddict) goto _output_error;
+
+ ZSTD_CCtx_reset(zc);
+ ZSTD_resetDStream(zd);
+ CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict));
+ CHECK_Z(ZSTD_initDStream_usingDDict(zd, ddict));
+ CHECK_Z(ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1U << kMaxWindowLog));
+ /* Test all values < 300 */
+ for (value = 0; value < 300; ++value) {
+ for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) {
+ CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value));
+ }
+ }
+ /* Test values 2^8 to 2^17 */
+ for (value = (1 << 8); value < (1 << 17); value <<= 1) {
+ for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) {
+ CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value));
+ CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value + (value >> 2)));
+ }
+ }
+ /* Test offset values up to the max window log */
+ for (value = 8; value <= kMaxWindowLog; ++value) {
+ CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, SEQ_gen_of, (1U << value) - 1));
+ }
+
+ CHECK_Z(SEQ_roundTrip(zc, zd, &xxh, NULL, 0, ZSTD_e_end));
+ CHECK(SEQ_digest(&seq) != XXH64_digest(&xxh), "SEQ XXH64 does not match");
+
+ ZSTD_freeCDict(cdict);
+ ZSTD_freeDDict(ddict);
+ }
+ DISPLAYLEVEL(3, "OK \n");
/* Overlen overwriting window data bug */
DISPLAYLEVEL(3, "test%3i : wildcopy doesn't overwrite potential match data : ", testNb++);
@@ -699,8 +883,6 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog)
return FUZ_rLogLength(seed, logLength);
}
-#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
-
/* Return value in range minVal <= v <= maxVal */
static U32 FUZ_randomClampedLength(U32* seed, U32 minVal, U32 maxVal)
{
@@ -708,14 +890,6 @@ static U32 FUZ_randomClampedLength(U32* seed, U32 minVal, U32 maxVal)
return (U32)((FUZ_rand(seed) % mod) + minVal);
}
-#define CHECK(cond, ...) { \
- if (cond) { \
- DISPLAY("Error => "); \
- DISPLAY(__VA_ARGS__); \
- DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); \
- goto _output_error; \
-} }
-
static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility, int bigTests)
{
U32 const maxSrcLog = bigTests ? 24 : 22;
@@ -734,7 +908,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
ZSTD_CStream* zc = ZSTD_createCStream(); /* will be re-created sometimes */
ZSTD_DStream* zd = ZSTD_createDStream(); /* will be re-created sometimes */
ZSTD_DStream* const zd_noise = ZSTD_createDStream();
- clock_t const startClock = clock();
+ UTIL_time_t const startClock = UTIL_getTime();
const BYTE* dict = NULL; /* can keep same dict on 2 consecutive tests */
size_t dictSize = 0;
U32 oldTestLog = 0;
@@ -764,7 +938,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
FUZ_rand(&coreSeed);
/* test loop */
- for ( ; (testNb <= nbTests) || (FUZ_GetClockSpan(startClock) < g_clockTime) ; testNb++ ) {
+ for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
U32 lseed;
const BYTE* srcBuffer;
size_t totalTestSize, totalGenSize, cSize;
@@ -832,10 +1006,11 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
{ size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
dict = srcBuffer + dictStart;
}
- { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize;
+ { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize);
params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
+ params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1;
CHECK_Z ( ZSTD_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) );
} }
@@ -846,7 +1021,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
for (n=0, cSize=0, totalTestSize=0 ; totalTestSize < maxTestSize ; n++) {
/* compress random chunks into randomly sized dst buffers */
{ size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
- size_t const srcSize = MIN (maxTestSize-totalTestSize, randomSrcSize);
+ size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
@@ -983,7 +1158,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
ZSTDMT_CCtx* zc = ZSTDMT_createCCtx(nbThreads); /* will be reset sometimes */
ZSTD_DStream* zd = ZSTD_createDStream(); /* will be reset sometimes */
ZSTD_DStream* const zd_noise = ZSTD_createDStream();
- clock_t const startClock = clock();
+ UTIL_time_t const startClock = UTIL_getTime();
const BYTE* dict=NULL; /* can keep same dict on 2 consecutive tests */
size_t dictSize = 0;
U32 oldTestLog = 0;
@@ -1014,7 +1189,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
FUZ_rand(&coreSeed);
/* test loop */
- for ( ; (testNb <= nbTests) || (FUZ_GetClockSpan(startClock) < g_clockTime) ; testNb++ ) {
+ for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
U32 lseed;
const BYTE* srcBuffer;
size_t totalTestSize, totalGenSize, cSize;
@@ -1086,17 +1261,17 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
{ size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
dict = srcBuffer + dictStart;
}
- { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize;
+ { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize);
- DISPLAYLEVEL(5, "Init with windowLog = %u and pledgedSrcSize = %u \n",
- params.cParams.windowLog, (U32)pledgedSrcSize);
+ DISPLAYLEVEL(5, "Init with windowLog = %u, pledgedSrcSize = %u, dictSize = %u \n",
+ params.cParams.windowLog, (U32)pledgedSrcSize, (U32)dictSize);
params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
- params.fParams.contentSizeFlag = pledgedSrcSize>0;
+ params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1;
DISPLAYLEVEL(5, "checksumFlag : %u \n", params.fParams.checksumFlag);
- CHECK_Z( ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) );
CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapSectionLog, FUZ_rand(&lseed) % 12) );
- CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_sectionSize, FUZ_rand(&lseed) % (2*maxTestSize+1)) );
+ CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_jobSize, FUZ_rand(&lseed) % (2*maxTestSize+1)) ); /* custome job size */
+ CHECK_Z( ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) );
} }
/* multi-segments compression test */
@@ -1113,9 +1288,9 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
outBuff.size = outBuff.pos + dstBuffSize;
- DISPLAYLEVEL(5, "Sending %u bytes to compress \n", (U32)srcSize);
+ DISPLAYLEVEL(6, "Sending %u bytes to compress \n", (U32)srcSize);
CHECK_Z( ZSTDMT_compressStream(zc, &outBuff, &inBuff) );
- DISPLAYLEVEL(5, "%u bytes read by ZSTDMT_compressStream \n", (U32)inBuff.pos);
+ DISPLAYLEVEL(6, "%u bytes read by ZSTDMT_compressStream \n", (U32)inBuff.pos);
XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
@@ -1162,10 +1337,10 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
inBuff.size = inBuff.pos + readCSrcSize;
outBuff.size = outBuff.pos + dstBuffSize;
- DISPLAYLEVEL(5, "ZSTD_decompressStream input %u bytes \n", (U32)readCSrcSize);
+ DISPLAYLEVEL(6, "ZSTD_decompressStream input %u bytes \n", (U32)readCSrcSize);
decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
- DISPLAYLEVEL(5, "inBuff.pos = %u \n", (U32)readCSrcSize);
+ DISPLAYLEVEL(6, "inBuff.pos = %u \n", (U32)readCSrcSize);
}
CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (U32)outBuff.pos, (U32)totalTestSize);
CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (U32)inBuff.pos, (U32)cSize);
@@ -1255,7 +1430,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
ZSTD_CCtx* zc = ZSTD_createCCtx(); /* will be reset sometimes */
ZSTD_DStream* zd = ZSTD_createDStream(); /* will be reset sometimes */
ZSTD_DStream* const zd_noise = ZSTD_createDStream();
- clock_t const startClock = clock();
+ UTIL_time_t const startClock = UTIL_getTime();
const BYTE* dict = NULL; /* can keep same dict on 2 consecutive tests */
size_t dictSize = 0;
U32 oldTestLog = 0;
@@ -1288,7 +1463,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
FUZ_rand(&coreSeed);
/* test loop */
- for ( ; (testNb <= nbTests) || (FUZ_GetClockSpan(startClock) < g_clockTime) ; testNb++ ) {
+ for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
U32 lseed;
const BYTE* srcBuffer;
size_t totalTestSize, totalGenSize, cSize;
@@ -1352,7 +1527,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
(MAX(testLog, dictLog) / 2))) +
1;
U32 const cLevel = MIN(cLevelCandidate, cLevelMax);
- DISPLAYLEVEL(5, "t%u: cLevel : %u \n", testNb, cLevel);
+ DISPLAYLEVEL(5, "t%u: base cLevel : %u \n", testNb, cLevel);
maxTestSize = FUZ_rLogLength(&lseed, testLog);
DISPLAYLEVEL(5, "t%u: maxTestSize : %u \n", testNb, (U32)maxTestSize);
oldTestLog = testLog;
@@ -1377,43 +1552,47 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
cParams = ZSTD_adjustCParams(cParams, 0, 0);
if (FUZ_rand(&lseed) & 1) {
+ DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog);
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_windowLog, cParams.windowLog, useOpaqueAPI) );
assert(cParams.windowLog >= ZSTD_WINDOWLOG_MIN); /* guaranteed by ZSTD_adjustCParams() */
windowLogMalus = (cParams.windowLog - ZSTD_WINDOWLOG_MIN) / 5;
- DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog);
}
if (FUZ_rand(&lseed) & 1) {
- CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_hashLog, cParams.hashLog, useOpaqueAPI) );
DISPLAYLEVEL(5, "t%u: hashLog : %u \n", testNb, cParams.hashLog);
+ CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_hashLog, cParams.hashLog, useOpaqueAPI) );
}
if (FUZ_rand(&lseed) & 1) {
- CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_chainLog, cParams.chainLog, useOpaqueAPI) );
DISPLAYLEVEL(5, "t%u: chainLog : %u \n", testNb, cParams.chainLog);
+ CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_chainLog, cParams.chainLog, useOpaqueAPI) );
}
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_searchLog, cParams.searchLog, useOpaqueAPI) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_minMatch, cParams.searchLength, useOpaqueAPI) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_targetLength, cParams.targetLength, useOpaqueAPI) );
/* mess with long distance matching parameters */
- if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, useOpaqueAPI) );
- if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), useOpaqueAPI) );
- if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), useOpaqueAPI) );
- if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_LDM_BUCKETSIZELOG_MAX), useOpaqueAPI) );
- if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashEveryLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN), useOpaqueAPI) );
+ if (bigTests) {
+ if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, useOpaqueAPI) );
+ if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), useOpaqueAPI) );
+ if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), useOpaqueAPI) );
+ if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_LDM_BUCKETSIZELOG_MAX), useOpaqueAPI) );
+ if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashEveryLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN), useOpaqueAPI) );
+ }
/* mess with frame parameters */
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_checksumFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_dictIDFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_contentSizeFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
- if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
- DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (U32)pledgedSrcSize);
+ if (FUZ_rand(&lseed) & 1) {
+ DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (U32)pledgedSrcSize);
+ CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
+ }
/* multi-threading parameters */
{ U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1;
U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1;
U32 const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax);
- CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_nbThreads, nbThreads, useOpaqueAPI) );
DISPLAYLEVEL(5, "t%u: nbThreads : %u \n", testNb, nbThreads);
+ CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_nbThreads, nbThreads, useOpaqueAPI) );
if (nbThreads > 1) {
U32 const jobLog = FUZ_rand(&lseed) % (testLog+1);
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_overlapSizeLog, FUZ_rand(&lseed) % 10, useOpaqueAPI) );
@@ -1421,10 +1600,11 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
}
}
- if (FUZ_rand(&lseed) & 1) CHECK_Z (setCCtxParameter(zc, cctxParams, ZSTD_p_forceMaxWindow, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
+ if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_forceMaxWindow, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
/* Apply parameters */
if (useOpaqueAPI) {
+ DISPLAYLEVEL(6," t%u: applying CCtxParams \n", testNb);
CHECK_Z (ZSTD_CCtx_setParametersUsingCCtxParams(zc, cctxParams) );
}
@@ -1464,8 +1644,8 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
outBuff.size = outBuff.pos + dstBuffSize;
CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, flush) );
- DISPLAYLEVEL(6, "compress consumed %u bytes (total : %u) \n",
- (U32)inBuff.pos, (U32)(totalTestSize + inBuff.pos));
+ DISPLAYLEVEL(6, "t%u: compress consumed %u bytes (total : %u) \n",
+ testNb, (U32)inBuff.pos, (U32)(totalTestSize + inBuff.pos));
XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
@@ -1593,24 +1773,23 @@ typedef enum { simple_api, mt_api, advanced_api } e_api;
int main(int argc, const char** argv)
{
- U32 seed=0;
- int seedset=0;
- int argNb;
+ U32 seed = 0;
+ int seedset = 0;
int nbTests = nbTestsDefault;
int testNb = 0;
int proba = FUZ_COMPRESSIBILITY_DEFAULT;
- int result=0;
+ int result = 0;
int mainPause = 0;
int bigTests = (sizeof(size_t) == 8);
e_api selected_api = simple_api;
const char* const programName = argv[0];
- ZSTD_customMem const customNULL = ZSTD_defaultCMem;
U32 useOpaqueAPI = 0;
+ int argNb;
/* Check command line */
for(argNb=1; argNb<argc; argNb++) {
const char* argument = argv[argNb];
- if(!argument) continue; /* Protection if argument empty */
+ assert(argument != NULL);
/* Parsing commands. Aggregated commands are allowed */
if (argument[0]=='-') {
@@ -1660,15 +1839,17 @@ int main(int argc, const char** argv)
g_clockTime += *argument - '0';
argument++;
}
- if (*argument=='m') g_clockTime *=60, argument++;
- if (*argument=='n') argument++;
- g_clockTime *= CLOCKS_PER_SEC;
+ if (*argument=='m') { /* -T1m == -T60 */
+ g_clockTime *=60, argument++;
+ if (*argument=='n') argument++; /* -T1mn == -T60 */
+ } else if (*argument=='s') argument++; /* -T10s == -T10 */
+ g_clockTime *= SEC_TO_MICRO;
break;
case 's': /* manually select seed */
argument++;
- seed=0;
seedset=1;
+ seed=0;
while ((*argument>='0') && (*argument<='9')) {
seed *= 10;
seed += *argument - '0';
@@ -1718,7 +1899,7 @@ int main(int argc, const char** argv)
if (nbTests<=0) nbTests=1;
if (testNb==0) {
- result = basicUnitTests(0, ((double)proba) / 100, customNULL); /* constant seed for predictability */
+ result = basicUnitTests(0, ((double)proba) / 100); /* constant seed for predictability */
}
if (!result) {
generated by cgit v1.2.3 (git 2.25.1) at 2025-08-13 16:45:29 +0000