vendor/zstd/1.3.3

1 files changed, 280 insertions, 99 deletions

diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 8b4c836940a63..5590c9af15f8e 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -24,7 +24,6 @@
 **************************************/
 #include <stdlib.h>       /* free */
 #include <stdio.h>        /* fgets, sscanf */
-#include <time.h>         /* clock_t, clock() */
 #include <string.h>       /* strcmp */
 #include <assert.h>       /* assert */
 #include "mem.h"
@@ -36,6 +35,8 @@
 #include "datagen.h"      /* RDG_genBuffer */
 #define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
 #include "xxhash.h"       /* XXH64_* */
+#include "seqgen.h"
+#include "util.h"
 
 
 /*-************************************
@@ -61,26 +62,24 @@ static const U32 prime32 = 2654435761U;
                                   if (g_displayLevel>=4) fflush(stderr); }
 static U32 g_displayLevel = 2;
 
+static const U64 g_refreshRate = SEC_TO_MICRO / 6;
+static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
+
 #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
-            if ((FUZ_GetClockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
-            { g_displayClock = clock(); DISPLAY(__VA_ARGS__); \
-              if (g_displayLevel>=4) fflush(stderr); } }
-static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6;
-static clock_t g_displayClock = 0;
+            if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
+            { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
+            if (g_displayLevel>=4) fflush(stderr); } }
 
-static clock_t g_clockTime = 0;
+static U64 g_clockTime = 0;
 
 
 /*-*******************************************************
 *  Fuzzer functions
 *********************************************************/
+#undef MIN
+#undef MAX
+#define MIN(a,b) ((a)<(b)?(a):(b))
 #define MAX(a,b) ((a)>(b)?(a):(b))
-
-static clock_t FUZ_GetClockSpan(clock_t clockStart)
-{
-    return clock() - clockStart;  /* works even when overflow. Max span ~ 30 mn */
-}
-
 /*! FUZ_rand() :
     @return : a 27 bits random value, from a 32-bits `seed`.
     `seed` is also modified */
@@ -96,15 +95,21 @@ unsigned int FUZ_rand(unsigned int* seedPtr)
     return rand32 >> 5;
 }
 
-#define CHECK_Z(f) {                                         \
-    size_t const err = f;                                    \
-    if (ZSTD_isError(err)) {                                 \
-        DISPLAY("Error => %s : %s ",                         \
-                #f, ZSTD_getErrorName(err));                 \
-        DISPLAY(" (seed %u, test nb %u)  \n", seed, testNb); \
+#define CHECK(cond, ...) {                                   \
+    if (cond) {                                              \
+        DISPLAY("Error => ");                                \
+        DISPLAY(__VA_ARGS__);                                \
+        DISPLAY(" (seed %u, test nb %u, line %u)  \n",       \
+                seed, testNb, __LINE__);                     \
         goto _output_error;                                  \
 }   }
 
+#define CHECK_Z(f) {                                         \
+    size_t const err = f;                                    \
+    CHECK(ZSTD_isError(err), "%s : %s ",                     \
+          #f, ZSTD_getErrorName(err));                       \
+}
+
 
 /*======================================================
 *   Basic Unit tests
@@ -144,12 +149,69 @@ static void FUZ_freeDictionary(buffer_t dict)
     free(dict.start);
 }
 
+/* Round trips data and updates xxh with the decompressed data produced */
+static size_t SEQ_roundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
+                            XXH64_state_t* xxh, void* data, size_t size,
+                            ZSTD_EndDirective endOp)
+{
+    static BYTE compressed[1024];
+    static BYTE uncompressed[1024];
+
+    ZSTD_inBuffer cin = {data, size, 0};
+    size_t cret;
+
+    do {
+        ZSTD_outBuffer cout = {compressed, sizeof(compressed), 0};
+        ZSTD_inBuffer din = {compressed, 0, 0};
+        ZSTD_outBuffer dout = {uncompressed, 0, 0};
+
+        cret = ZSTD_compress_generic(cctx, &cout, &cin, endOp);
+        if (ZSTD_isError(cret))
+            return cret;
+
+        din.size = cout.pos;
+        while (din.pos < din.size || (endOp == ZSTD_e_end && cret == 0)) {
+            size_t dret;
+
+            dout.pos = 0;
+            dout.size = sizeof(uncompressed);
+            dret = ZSTD_decompressStream(dctx, &dout, &din);
+            if (ZSTD_isError(dret))
+                return dret;
+            XXH64_update(xxh, dout.dst, dout.pos);
+            if (dret == 0)
+                break;
+        }
+    } while (cin.pos < cin.size || (endOp != ZSTD_e_continue && cret != 0));
+    return 0;
+}
+
+/* Generates some data and round trips it */
+static size_t SEQ_generateRoundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
+                                    XXH64_state_t* xxh, SEQ_stream* seq,
+                                    SEQ_gen_type type, unsigned value)
+{
+    static BYTE data[1024];
+    size_t gen;
+
+    do {
+        SEQ_outBuffer sout = {data, sizeof(data), 0};
+        size_t ret;
+        gen = SEQ_gen(seq, type, value, &sout);
+
+        ret = SEQ_roundTrip(cctx, dctx, xxh, sout.dst, sout.pos, ZSTD_e_continue);
+        if (ZSTD_isError(ret))
+            return ret;
+    } while (gen != 0);
+
+    return 0;
+}
 
-static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem)
+static int basicUnitTests(U32 seed, double compressibility)
 {
     size_t const CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
     void* CNBuffer = malloc(CNBufferSize);
-    size_t const skippableFrameSize = 11;
+    size_t const skippableFrameSize = 200 KB;
     size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
     void* compressedBuffer = malloc(compressedBufferSize);
     size_t const decodedBufferSize = CNBufferSize;
@@ -157,8 +219,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     size_t cSize;
     int testResult = 0;
     U32 testNb = 1;
-    ZSTD_CStream* zc = ZSTD_createCStream_advanced(customMem);
-    ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem);
+    ZSTD_CStream* zc = ZSTD_createCStream();
+    ZSTD_DStream* zd = ZSTD_createDStream();
     ZSTDMT_CCtx* mtctx = ZSTDMT_createCCtx(2);
 
     ZSTD_inBuffer  inBuff, inBuff2;
@@ -183,16 +245,32 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     }
     dictID = ZDICT_getDictID(dictionary.start, dictionary.filled);
 
+    /* Basic compression test */
+    DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
+    CHECK_Z( ZSTD_initCStream(zc, 1 /* cLevel */) );
+    outBuff.dst = (char*)(compressedBuffer);
+    outBuff.size = compressedBufferSize;
+    outBuff.pos = 0;
+    inBuff.src = CNBuffer;
+    inBuff.size = CNBufferSize;
+    inBuff.pos = 0;
+    CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
+    if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
+    { size_t const r = ZSTD_endStream(zc, &outBuff);
+      if (r != 0) goto _output_error; }  /* error, or some data not flushed */
+    DISPLAYLEVEL(3, "OK (%u bytes)\n", (U32)outBuff.pos);
+
     /* generate skippable frame */
     MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
     MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize);
     cSize = skippableFrameSize + 8;
 
-    /* Basic compression test */
-    DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
-    CHECK_Z( ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1) );
+    /* Basic compression test using dict */
+    DISPLAYLEVEL(3, "test%3i : skipframe + compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
+    CHECK_Z( ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1 /* cLevel */) );
     outBuff.dst = (char*)(compressedBuffer)+cSize;
-    outBuff.size = compressedBufferSize;
+    assert(compressedBufferSize > cSize);
+    outBuff.size = compressedBufferSize - cSize;
     outBuff.pos = 0;
     inBuff.src = CNBuffer;
     inBuff.size = CNBufferSize;
@@ -366,7 +444,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     /* Complex context re-use scenario */
     DISPLAYLEVEL(3, "test%3i : context re-use : ", testNb++);
     ZSTD_freeCStream(zc);
-    zc = ZSTD_createCStream_advanced(customMem);
+    zc = ZSTD_createCStream();
     if (zc==NULL) goto _output_error;   /* memory allocation issue */
     /* use 1 */
     {   size_t const inSize = 513;
@@ -408,6 +486,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     DISPLAYLEVEL(3, "test%3i : digested dictionary : ", testNb++);
     {   ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, 1 /*byRef*/ );
         size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict);
+        DISPLAYLEVEL(5, "ZSTD_initCStream_usingCDict result : %u ", (U32)initError);
         if (ZSTD_isError(initError)) goto _output_error;
         cSize = 0;
         outBuff.dst = compressedBuffer;
@@ -416,10 +495,13 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
         inBuff.src = CNBuffer;
         inBuff.size = CNBufferSize;
         inBuff.pos = 0;
+        DISPLAYLEVEL(5, "- starting ZSTD_compressStream ");
         CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
         if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
-        { size_t const r = ZSTD_endStream(zc, &outBuff);
-          if (r != 0) goto _output_error; }  /* error, or some data not flushed */
+        {   size_t const r = ZSTD_endStream(zc, &outBuff);
+            DISPLAYLEVEL(5, "- ZSTD_endStream result : %u ", (U32)r);
+            if (r != 0) goto _output_error;  /* error, or some data not flushed */
+        }
         cSize = outBuff.pos;
         ZSTD_freeCDict(cdict);
         DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);
@@ -442,12 +524,12 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     {   ZSTD_DDict* const ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
         size_t const initError = ZSTD_initDStream_usingDDict(zd, ddict);
         if (ZSTD_isError(initError)) goto _output_error;
-        inBuff.src = compressedBuffer;
-        inBuff.size = cSize;
-        inBuff.pos = 0;
         outBuff.dst = decodedBuffer;
         outBuff.size = CNBufferSize;
         outBuff.pos = 0;
+        inBuff.src = compressedBuffer;
+        inBuff.size = cSize;
+        inBuff.pos = 0;
         { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
           if (r != 0) goto _output_error; }  /* should reach end of frame == 0; otherwise, some data left, or an error */
         if (outBuff.pos != CNBufferSize) goto _output_error;   /* should regenerate the same amount */
@@ -466,12 +548,12 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     DISPLAYLEVEL(3, "test%3i : maxWindowSize < frame requirement : ", testNb++);
     ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
     CHECK_Z( ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1000) );  /* too small limit */
-    inBuff.src = compressedBuffer;
-    inBuff.size = cSize;
-    inBuff.pos = 0;
     outBuff.dst = decodedBuffer;
     outBuff.size = CNBufferSize;
     outBuff.pos = 0;
+    inBuff.src = compressedBuffer;
+    inBuff.size = cSize;
+    inBuff.pos = 0;
     { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
       if (!ZSTD_isError(r)) goto _output_error;  /* must fail : frame requires > 100 bytes */
       DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); }
@@ -479,7 +561,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_usingCDict_advanced with masked dictID : ", testNb++);
     {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictionary.filled);
         ZSTD_frameParameters const fParams = { 1 /* contentSize */, 1 /* checksum */, 1 /* noDictID */};
-        ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dm_auto, cParams, customMem);
+        ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dm_auto, cParams, ZSTD_defaultCMem);
         size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, fParams, CNBufferSize);
         if (ZSTD_isError(initError)) goto _output_error;
         cSize = 0;
@@ -558,14 +640,14 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced with pledgedSrcSize=0 and dict : ", testNb++);
     {   ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
         params.fParams.contentSizeFlag = 1;
-        CHECK_Z( ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0) );
+        CHECK_Z( ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0 /* pledgedSrcSize==0 means "empty" when params.fParams.contentSizeFlag is set */) );
     } /* cstream advanced shall write content size = 0 */
-    inBuff.src = CNBuffer;
-    inBuff.size = 0;
-    inBuff.pos = 0;
     outBuff.dst = compressedBuffer;
     outBuff.size = compressedBufferSize;
     outBuff.pos = 0;
+    inBuff.src = CNBuffer;
+    inBuff.size = 0;
+    inBuff.pos = 0;
     CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
     if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
     cSize = outBuff.pos;
@@ -589,12 +671,12 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
 
     ZSTD_resetCStream(zc, 0); /* resetCStream should treat 0 as unknown */
-    inBuff.src = CNBuffer;
-    inBuff.size = 0;
-    inBuff.pos = 0;
     outBuff.dst = compressedBuffer;
     outBuff.size = compressedBufferSize;
     outBuff.pos = 0;
+    inBuff.src = CNBuffer;
+    inBuff.size = 0;
+    inBuff.pos = 0;
     CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
     if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
     cSize = outBuff.pos;
@@ -606,7 +688,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     {   ZSTD_parameters const params = ZSTD_getParams(1, 0, 0);
         CHECK_Z( ZSTDMT_initCStream_advanced(mtctx, CNBuffer, dictSize, params, CNBufferSize) );
     }
-    outBuff.dst = (char*)(compressedBuffer);
+    outBuff.dst = compressedBuffer;
     outBuff.size = compressedBufferSize;
     outBuff.pos = 0;
     inBuff.src = CNBuffer;
@@ -618,6 +700,108 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
       if (r != 0) goto _output_error; }  /* error, or some data not flushed */
     DISPLAYLEVEL(3, "OK \n");
 
+    /* Complex multithreading + dictionary test */
+    {   U32 const nbThreads = 2;
+        size_t const jobSize = 4 * 1 MB;
+        size_t const srcSize = jobSize * nbThreads;  /* we want each job to have predictable size */
+        size_t const segLength = 2 KB;
+        size_t const offset = 600 KB;   /* must be larger than window defined in cdict */
+        size_t const start = jobSize + (offset-1);
+        const BYTE* const srcToCopy = (const BYTE*)CNBuffer + start;
+        BYTE* const dst = (BYTE*)CNBuffer + start - offset;
+        DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads + dictionary : ", testNb++, (U32)srcSize);
+        CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_compressionLevel, 3) );
+        CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_nbThreads, 2) );
+        CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_jobSize, jobSize) );
+        assert(start > offset);
+        assert(start + segLength < COMPRESSIBLE_NOISE_LENGTH);
+        memcpy(dst, srcToCopy, segLength);   /* create a long repetition at long distance for job 2 */
+        outBuff.dst = compressedBuffer;
+        outBuff.size = compressedBufferSize;
+        outBuff.pos = 0;
+        inBuff.src = CNBuffer;
+        inBuff.size = srcSize; assert(srcSize < COMPRESSIBLE_NOISE_LENGTH);
+        inBuff.pos = 0;
+    }
+    {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 4 KB, dictionary.filled);   /* intentionnally lies on estimatedSrcSize, to push cdict into targeting a small window size */
+        ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dm_fullDict, cParams, ZSTD_defaultCMem);
+        DISPLAYLEVEL(5, "cParams.windowLog = %u : ", cParams.windowLog);
+        CHECK_Z( ZSTD_CCtx_refCDict(zc, cdict) );
+        CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end) );
+        CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) );  /* do not keep a reference to cdict, as its lifetime ends */
+        ZSTD_freeCDict(cdict);
+    }
+    if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
+    cSize = outBuff.pos;
+    DISPLAYLEVEL(3, "OK \n");
+
+    DISPLAYLEVEL(3, "test%3i : decompress large frame created from multiple threads + dictionary : ", testNb++);
+    {   ZSTD_DStream* const dstream = ZSTD_createDCtx();
+        ZSTD_frameHeader zfh;
+        ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize);
+        DISPLAYLEVEL(5, "frame windowsize = %u : ", (U32)zfh.windowSize);
+        outBuff.dst = decodedBuffer;
+        outBuff.size = CNBufferSize;
+        outBuff.pos = 0;
+        inBuff.src = compressedBuffer;
+        inBuff.pos = 0;
+        CHECK_Z( ZSTD_initDStream_usingDict(dstream, dictionary.start, dictionary.filled) );
+        inBuff.size = 1;  /* avoid shortcut to single-pass mode */
+        CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) );
+        inBuff.size = cSize;
+        CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) );
+        if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
+        ZSTD_freeDStream(dstream);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
+    DISPLAYLEVEL(3, "test%3i : check dictionary FSE tables can represent every code : ", testNb++);
+    {   unsigned const kMaxWindowLog = 24;
+        unsigned value;
+        ZSTD_compressionParameters cParams = ZSTD_getCParams(3, 1U << kMaxWindowLog, 1024);
+        ZSTD_CDict* cdict;
+        ZSTD_DDict* ddict;
+        SEQ_stream seq = SEQ_initStream(0x87654321);
+        SEQ_gen_type type;
+        XXH64_state_t xxh;
+
+        XXH64_reset(&xxh, 0);
+        cParams.windowLog = kMaxWindowLog;
+        cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dm_fullDict, cParams, ZSTD_defaultCMem);
+        ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
+
+        if (!cdict || !ddict) goto _output_error;
+
+        ZSTD_CCtx_reset(zc);
+        ZSTD_resetDStream(zd);
+        CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict));
+        CHECK_Z(ZSTD_initDStream_usingDDict(zd, ddict));
+        CHECK_Z(ZSTD_setDStreamParameter(zd, DStream_p_maxWindowSize, 1U << kMaxWindowLog));
+        /* Test all values < 300 */
+        for (value = 0; value < 300; ++value) {
+            for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) {
+                CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value));
+            }
+        }
+        /* Test values 2^8 to 2^17 */
+        for (value = (1 << 8); value < (1 << 17); value <<= 1) {
+            for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) {
+                CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value));
+                CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value + (value >> 2)));
+            }
+        }
+        /* Test offset values up to the max window log */
+        for (value = 8; value <= kMaxWindowLog; ++value) {
+            CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, SEQ_gen_of, (1U << value) - 1));
+        }
+
+        CHECK_Z(SEQ_roundTrip(zc, zd, &xxh, NULL, 0, ZSTD_e_end));
+        CHECK(SEQ_digest(&seq) != XXH64_digest(&xxh), "SEQ XXH64 does not match");
+
+        ZSTD_freeCDict(cdict);
+        ZSTD_freeDDict(ddict);
+    }
+    DISPLAYLEVEL(3, "OK \n");
 
     /* Overlen overwriting window data bug */
     DISPLAYLEVEL(3, "test%3i : wildcopy doesn't overwrite potential match data : ", testNb++);
@@ -699,8 +883,6 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog)
     return FUZ_rLogLength(seed, logLength);
 }
 
-#define MIN(a,b)   ( (a) < (b) ? (a) : (b) )
-
 /* Return value in range minVal <= v <= maxVal */
 static U32 FUZ_randomClampedLength(U32* seed, U32 minVal, U32 maxVal)
 {
@@ -708,14 +890,6 @@ static U32 FUZ_randomClampedLength(U32* seed, U32 minVal, U32 maxVal)
     return (U32)((FUZ_rand(seed) % mod) + minVal);
 }
 
-#define CHECK(cond, ...) {                                   \
-    if (cond) {                                              \
-        DISPLAY("Error => ");                                \
-        DISPLAY(__VA_ARGS__);                                \
-        DISPLAY(" (seed %u, test nb %u)  \n", seed, testNb); \
-        goto _output_error;                                  \
-}   }
-
 static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility, int bigTests)
 {
     U32 const maxSrcLog = bigTests ? 24 : 22;
@@ -734,7 +908,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
     ZSTD_CStream* zc = ZSTD_createCStream();   /* will be re-created sometimes */
     ZSTD_DStream* zd = ZSTD_createDStream();   /* will be re-created sometimes */
     ZSTD_DStream* const zd_noise = ZSTD_createDStream();
-    clock_t const startClock = clock();
+    UTIL_time_t const startClock = UTIL_getTime();
     const BYTE* dict = NULL;  /* can keep same dict on 2 consecutive tests */
     size_t dictSize = 0;
     U32 oldTestLog = 0;
@@ -764,7 +938,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
         FUZ_rand(&coreSeed);
 
     /* test loop */
-    for ( ; (testNb <= nbTests) || (FUZ_GetClockSpan(startClock) < g_clockTime) ; testNb++ ) {
+    for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
         U32 lseed;
         const BYTE* srcBuffer;
         size_t totalTestSize, totalGenSize, cSize;
@@ -832,10 +1006,11 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
             {   size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
                 dict = srcBuffer + dictStart;
             }
-            {   U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize;
+            {   U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
                 ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize);
                 params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
                 params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
+                params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1;
                 CHECK_Z ( ZSTD_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) );
         }   }
 
@@ -846,7 +1021,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
             for (n=0, cSize=0, totalTestSize=0 ; totalTestSize < maxTestSize ; n++) {
                 /* compress random chunks into randomly sized dst buffers */
                 {   size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
-                    size_t const srcSize = MIN (maxTestSize-totalTestSize, randomSrcSize);
+                    size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
                     size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
                     size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
                     size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
@@ -983,7 +1158,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
     ZSTDMT_CCtx* zc = ZSTDMT_createCCtx(nbThreads);   /* will be reset sometimes */
     ZSTD_DStream* zd = ZSTD_createDStream();   /* will be reset sometimes */
     ZSTD_DStream* const zd_noise = ZSTD_createDStream();
-    clock_t const startClock = clock();
+    UTIL_time_t const startClock = UTIL_getTime();
     const BYTE* dict=NULL;   /* can keep same dict on 2 consecutive tests */
     size_t dictSize = 0;
     U32 oldTestLog = 0;
@@ -1014,7 +1189,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
         FUZ_rand(&coreSeed);
 
     /* test loop */
-    for ( ; (testNb <= nbTests) || (FUZ_GetClockSpan(startClock) < g_clockTime) ; testNb++ ) {
+    for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
         U32 lseed;
         const BYTE* srcBuffer;
         size_t totalTestSize, totalGenSize, cSize;
@@ -1086,17 +1261,17 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
             {   size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
                 dict = srcBuffer + dictStart;
             }
-            {   U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize;
+            {   U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
                 ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize);
-                DISPLAYLEVEL(5, "Init with windowLog = %u and pledgedSrcSize = %u \n",
-                    params.cParams.windowLog, (U32)pledgedSrcSize);
+                DISPLAYLEVEL(5, "Init with windowLog = %u, pledgedSrcSize = %u, dictSize = %u \n",
+                    params.cParams.windowLog, (U32)pledgedSrcSize, (U32)dictSize);
                 params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
                 params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
-                params.fParams.contentSizeFlag = pledgedSrcSize>0;
+                params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1;
                 DISPLAYLEVEL(5, "checksumFlag : %u \n", params.fParams.checksumFlag);
-                CHECK_Z( ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) );
                 CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapSectionLog, FUZ_rand(&lseed) % 12) );
-                CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_sectionSize, FUZ_rand(&lseed) % (2*maxTestSize+1)) );
+                CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_jobSize, FUZ_rand(&lseed) % (2*maxTestSize+1)) );   /* custome job size */
+                CHECK_Z( ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) );
         }   }
 
         /* multi-segments compression test */
@@ -1113,9 +1288,9 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
                     ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
                     outBuff.size = outBuff.pos + dstBuffSize;
 
-                    DISPLAYLEVEL(5, "Sending %u bytes to compress \n", (U32)srcSize);
+                    DISPLAYLEVEL(6, "Sending %u bytes to compress \n", (U32)srcSize);
                     CHECK_Z( ZSTDMT_compressStream(zc, &outBuff, &inBuff) );
-                    DISPLAYLEVEL(5, "%u bytes read by ZSTDMT_compressStream \n", (U32)inBuff.pos);
+                    DISPLAYLEVEL(6, "%u bytes read by ZSTDMT_compressStream \n", (U32)inBuff.pos);
 
                     XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
                     memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
@@ -1162,10 +1337,10 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
                 size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
                 inBuff.size = inBuff.pos + readCSrcSize;
                 outBuff.size = outBuff.pos + dstBuffSize;
-                DISPLAYLEVEL(5, "ZSTD_decompressStream input %u bytes \n", (U32)readCSrcSize);
+                DISPLAYLEVEL(6, "ZSTD_decompressStream input %u bytes \n", (U32)readCSrcSize);
                 decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
                 CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
-                DISPLAYLEVEL(5, "inBuff.pos = %u \n", (U32)readCSrcSize);
+                DISPLAYLEVEL(6, "inBuff.pos = %u \n", (U32)readCSrcSize);
             }
             CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (U32)outBuff.pos, (U32)totalTestSize);
             CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (U32)inBuff.pos, (U32)cSize);
@@ -1255,7 +1430,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
     ZSTD_CCtx* zc = ZSTD_createCCtx();   /* will be reset sometimes */
     ZSTD_DStream* zd = ZSTD_createDStream();   /* will be reset sometimes */
     ZSTD_DStream* const zd_noise = ZSTD_createDStream();
-    clock_t const startClock = clock();
+    UTIL_time_t const startClock = UTIL_getTime();
     const BYTE* dict = NULL;   /* can keep same dict on 2 consecutive tests */
     size_t dictSize = 0;
     U32 oldTestLog = 0;
@@ -1288,7 +1463,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
         FUZ_rand(&coreSeed);
 
     /* test loop */
-    for ( ; (testNb <= nbTests) || (FUZ_GetClockSpan(startClock) < g_clockTime) ; testNb++ ) {
+    for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
         U32 lseed;
         const BYTE* srcBuffer;
         size_t totalTestSize, totalGenSize, cSize;
@@ -1352,7 +1527,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
                                (MAX(testLog, dictLog) / 2))) +
                                1;
             U32 const cLevel = MIN(cLevelCandidate, cLevelMax);
-            DISPLAYLEVEL(5, "t%u: cLevel : %u \n", testNb, cLevel);
+            DISPLAYLEVEL(5, "t%u: base cLevel : %u \n", testNb, cLevel);
             maxTestSize = FUZ_rLogLength(&lseed, testLog);
             DISPLAYLEVEL(5, "t%u: maxTestSize : %u \n", testNb, (U32)maxTestSize);
             oldTestLog = testLog;
@@ -1377,43 +1552,47 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
                 cParams = ZSTD_adjustCParams(cParams, 0, 0);
 
                 if (FUZ_rand(&lseed) & 1) {
+                    DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog);
                     CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_windowLog, cParams.windowLog, useOpaqueAPI) );
                     assert(cParams.windowLog >= ZSTD_WINDOWLOG_MIN);   /* guaranteed by ZSTD_adjustCParams() */
                     windowLogMalus = (cParams.windowLog - ZSTD_WINDOWLOG_MIN) / 5;
-                    DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog);
                 }
                 if (FUZ_rand(&lseed) & 1) {
-                    CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_hashLog, cParams.hashLog, useOpaqueAPI) );
                     DISPLAYLEVEL(5, "t%u: hashLog : %u \n", testNb, cParams.hashLog);
+                    CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_hashLog, cParams.hashLog, useOpaqueAPI) );
                 }
                 if (FUZ_rand(&lseed) & 1) {
-                    CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_chainLog, cParams.chainLog, useOpaqueAPI) );
                     DISPLAYLEVEL(5, "t%u: chainLog : %u \n", testNb, cParams.chainLog);
+                    CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_chainLog, cParams.chainLog, useOpaqueAPI) );
                 }
                 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_searchLog, cParams.searchLog, useOpaqueAPI) );
                 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_minMatch, cParams.searchLength, useOpaqueAPI) );
                 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_targetLength, cParams.targetLength, useOpaqueAPI) );
 
                 /* mess with long distance matching parameters */
-                if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, useOpaqueAPI) );
-                if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), useOpaqueAPI) );
-                if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), useOpaqueAPI) );
-                if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_LDM_BUCKETSIZELOG_MAX), useOpaqueAPI) );
-                if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashEveryLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN), useOpaqueAPI) );
+                if (bigTests) {
+                    if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, useOpaqueAPI) );
+                    if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), useOpaqueAPI) );
+                    if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), useOpaqueAPI) );
+                    if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_LDM_BUCKETSIZELOG_MAX), useOpaqueAPI) );
+                    if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashEveryLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN), useOpaqueAPI) );
+                }
 
                 /* mess with frame parameters */
                 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_checksumFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
                 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_dictIDFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
                 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_contentSizeFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
-                if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
-                DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (U32)pledgedSrcSize);
+                if (FUZ_rand(&lseed) & 1) {
+                    DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (U32)pledgedSrcSize);
+                    CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
+                }
 
                 /* multi-threading parameters */
                 {   U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1;
                     U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1;
                     U32 const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax);
-                    CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_nbThreads, nbThreads, useOpaqueAPI) );
                     DISPLAYLEVEL(5, "t%u: nbThreads : %u \n", testNb, nbThreads);
+                    CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_nbThreads, nbThreads, useOpaqueAPI) );
                     if (nbThreads > 1) {
                         U32 const jobLog = FUZ_rand(&lseed) % (testLog+1);
                         CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_overlapSizeLog, FUZ_rand(&lseed) % 10, useOpaqueAPI) );
@@ -1421,10 +1600,11 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
                     }
                 }
 
-                if (FUZ_rand(&lseed) & 1) CHECK_Z (setCCtxParameter(zc, cctxParams, ZSTD_p_forceMaxWindow, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
+                if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_forceMaxWindow, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
 
                 /* Apply parameters */
                 if (useOpaqueAPI) {
+                    DISPLAYLEVEL(6," t%u: applying CCtxParams \n", testNb);
                     CHECK_Z (ZSTD_CCtx_setParametersUsingCCtxParams(zc, cctxParams) );
                 }
 
@@ -1464,8 +1644,8 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
                 outBuff.size = outBuff.pos + dstBuffSize;
 
                 CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, flush) );
-                DISPLAYLEVEL(6, "compress consumed %u bytes (total : %u) \n",
-                    (U32)inBuff.pos, (U32)(totalTestSize + inBuff.pos));
+                DISPLAYLEVEL(6, "t%u: compress consumed %u bytes (total : %u) \n",
+                    testNb, (U32)inBuff.pos, (U32)(totalTestSize + inBuff.pos));
 
                 XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
                 memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
@@ -1593,24 +1773,23 @@ typedef enum { simple_api, mt_api, advanced_api } e_api;
 
 int main(int argc, const char** argv)
 {
-    U32 seed=0;
-    int seedset=0;
-    int argNb;
+    U32 seed = 0;
+    int seedset = 0;
     int nbTests = nbTestsDefault;
     int testNb = 0;
     int proba = FUZ_COMPRESSIBILITY_DEFAULT;
-    int result=0;
+    int result = 0;
     int mainPause = 0;
     int bigTests = (sizeof(size_t) == 8);
     e_api selected_api = simple_api;
     const char* const programName = argv[0];
-    ZSTD_customMem const customNULL = ZSTD_defaultCMem;
     U32 useOpaqueAPI = 0;
+    int argNb;
 
     /* Check command line */
     for(argNb=1; argNb<argc; argNb++) {
         const char* argument = argv[argNb];
-        if(!argument) continue;   /* Protection if argument empty */
+        assert(argument != NULL);
 
         /* Parsing commands. Aggregated commands are allowed */
         if (argument[0]=='-') {
@@ -1660,15 +1839,17 @@ int main(int argc, const char** argv)
                         g_clockTime += *argument - '0';
                         argument++;
                     }
-                    if (*argument=='m') g_clockTime *=60, argument++;
-                    if (*argument=='n') argument++;
-                    g_clockTime *= CLOCKS_PER_SEC;
+                    if (*argument=='m') {    /* -T1m == -T60 */
+                        g_clockTime *=60, argument++;
+                        if (*argument=='n') argument++; /* -T1mn == -T60 */
+                    } else if (*argument=='s') argument++; /* -T10s == -T10 */
+                    g_clockTime *= SEC_TO_MICRO;
                     break;
 
                 case 's':   /* manually select seed */
                     argument++;
-                    seed=0;
                     seedset=1;
+                    seed=0;
                     while ((*argument>='0') && (*argument<='9')) {
                         seed *= 10;
                         seed += *argument - '0';
@@ -1718,7 +1899,7 @@ int main(int argc, const char** argv)
     if (nbTests<=0) nbTests=1;
 
     if (testNb==0) {
-        result = basicUnitTests(0, ((double)proba) / 100, customNULL);  /* constant seed for predictability */
+        result = basicUnitTests(0, ((double)proba) / 100);  /* constant seed for predictability */
     }
 
     if (!result) {