diff options
Diffstat (limited to 'programs')
| -rw-r--r-- | programs/Makefile | 2 | ||||
| -rw-r--r-- | programs/README.md | 92 | ||||
| -rw-r--r-- | programs/bench.c | 101 | ||||
| -rw-r--r-- | programs/bench.h | 2 | ||||
| -rw-r--r-- | programs/fileio.c | 272 | ||||
| -rw-r--r-- | programs/fileio.h | 2 | ||||
| -rw-r--r-- | programs/platform.h | 5 | ||||
| -rw-r--r-- | programs/util.h | 18 | ||||
| -rw-r--r-- | programs/zstd.1 | 14 | ||||
| -rw-r--r-- | programs/zstd.1.md | 57 | ||||
| -rw-r--r-- | programs/zstdcli.c | 146 |
11 files changed, 436 insertions, 275 deletions
diff --git a/programs/Makefile b/programs/Makefile index 51888d3ef5cfb..be666b4ef0547 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -157,6 +157,8 @@ endif zstd-release: DEBUGFLAGS := zstd-release: zstd +zstd32 : CPPFLAGS += $(THREAD_CPP) +zstd32 : LDFLAGS += $(THREAD_LD) zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) zstd32 : $(ZSTDLIB_FILES) zstdcli.c fileio.c bench.c datagen.c dibio.c ifneq (,$(filter Windows%,$(OS))) diff --git a/programs/README.md b/programs/README.md index 2da9a6db1261e..a308fccf9ea3b 100644 --- a/programs/README.md +++ b/programs/README.md @@ -11,61 +11,74 @@ There are however other Makefile targets that create different variations of CLI #### Compilation variables -`zstd` scope can be altered by modifying the following compilation variables : +`zstd` scope can be altered by modifying the following `make` variables : - __HAVE_THREAD__ : multithreading is automatically enabled when `pthread` is detected. - It's possible to disable multithread support, by setting HAVE_THREAD=0 . - Example : make zstd HAVE_THREAD=0 - It's also possible to force compilation with multithread support, using HAVE_THREAD=1. - In which case, linking stage will fail if `pthread` library cannot be found. - This might be useful to prevent silent feature disabling. + It's possible to disable multithread support, by setting `HAVE_THREAD=0`. + Example : `make zstd HAVE_THREAD=0` + It's also possible to force multithread support, using `HAVE_THREAD=1`. + In which case, linking stage will fail if neither `pthread` nor `windows.h` library can be found. + This is useful to ensure this feature is not silently disabled. + +- __ZSTD_LEGACY_SUPPORT__ : `zstd` can decompress files compressed by older versions of `zstd`. + Starting v0.8.0, all versions of `zstd` produce frames compliant with the [specification](../doc/zstd_compression_format.md), and are therefore compatible. + But older versions (< v0.8.0) produced different, incompatible, frames. + By default, `zstd` supports decoding legacy formats >= v0.4.0 (`ZSTD_LEGACY_SUPPORT=4`). + This can be altered by modifying this compilation variable. + `ZSTD_LEGACY_SUPPORT=1` means "support all formats >= v0.1.0". + `ZSTD_LEGACY_SUPPORT=2` means "support all formats >= v0.2.0", and so on. + `ZSTD_LEGACY_SUPPORT=0` means _DO NOT_ support any legacy format. + if `ZSTD_LEGACY_SUPPORT >= 8`, it's the same as `0`, since there is no legacy format after `7`. + Note : `zstd` only supports decoding older formats, and cannot generate any legacy format. - __HAVE_ZLIB__ : `zstd` can compress and decompress files in `.gz` format. This is ordered through command `--format=gzip`. Alternatively, symlinks named `gzip` or `gunzip` will mimic intended behavior. `.gz` support is automatically enabled when `zlib` library is detected at build time. - It's possible to disable `.gz` support, by setting HAVE_ZLIB=0. - Example : make zstd HAVE_ZLIB=0 - It's also possible to force compilation with zlib support, using HAVE_ZLIB=1. + It's possible to disable `.gz` support, by setting `HAVE_ZLIB=0`. + Example : `make zstd HAVE_ZLIB=0` + It's also possible to force compilation with zlib support, `using HAVE_ZLIB=1`. In which case, linking stage will fail if `zlib` library cannot be found. - This might be useful to prevent silent feature disabling. + This is useful to prevent silent feature disabling. - __HAVE_LZMA__ : `zstd` can compress and decompress files in `.xz` and `.lzma` formats. This is ordered through commands `--format=xz` and `--format=lzma` respectively. Alternatively, symlinks named `xz`, `unxz`, `lzma`, or `unlzma` will mimic intended behavior. `.xz` and `.lzma` support is automatically enabled when `lzma` library is detected at build time. - It's possible to disable `.xz` and `.lzma` support, by setting HAVE_LZMA=0 . - Example : make zstd HAVE_LZMA=0 - It's also possible to force compilation with lzma support, using HAVE_LZMA=1. + It's possible to disable `.xz` and `.lzma` support, by setting `HAVE_LZMA=0` . + Example : `make zstd HAVE_LZMA=0` + It's also possible to force compilation with lzma support, using `HAVE_LZMA=1`. In which case, linking stage will fail if `lzma` library cannot be found. - This might be useful to prevent silent feature disabling. + This is useful to prevent silent feature disabling. - __HAVE_LZ4__ : `zstd` can compress and decompress files in `.lz4` formats. This is ordered through commands `--format=lz4`. Alternatively, symlinks named `lz4`, or `unlz4` will mimic intended behavior. `.lz4` support is automatically enabled when `lz4` library is detected at build time. - It's possible to disable `.lz4` support, by setting HAVE_LZ4=0 . - Example : make zstd HAVE_LZ4=0 - It's also possible to force compilation with lz4 support, using HAVE_LZ4=1. + It's possible to disable `.lz4` support, by setting `HAVE_LZ4=0` . + Example : `make zstd HAVE_LZ4=0` + It's also possible to force compilation with lz4 support, using `HAVE_LZ4=1`. In which case, linking stage will fail if `lz4` library cannot be found. - This might be useful to prevent silent feature disabling. - -- __ZSTD_LEGACY_SUPPORT__ : `zstd` can decompress files compressed by older versions of `zstd`. - Starting v0.8.0, all versions of `zstd` produce frames compliant with the [specification](../doc/zstd_compression_format.md), and are therefore compatible. - But older versions (< v0.8.0) produced different, incompatible, frames. - By default, `zstd` supports decoding legacy formats >= v0.4.0 (`ZSTD_LEGACY_SUPPORT=4`). - This can be altered by modifying this compilation variable. - `ZSTD_LEGACY_SUPPORT=1` means "support all formats >= v0.1.0". - `ZSTD_LEGACY_SUPPORT=2` means "support all formats >= v0.2.0", and so on. - `ZSTD_LEGACY_SUPPORT=0` means _DO NOT_ support any legacy format. - if `ZSTD_LEGACY_SUPPORT >= 8`, it's the same as `0`, since there is no legacy format after `7`. - Note : `zstd` only supports decoding older formats, and cannot generate any legacy format. + This is useful to prevent silent feature disabling. #### Aggregation of parameters CLI supports aggregation of parameters i.e. `-b1`, `-e18`, and `-i1` can be joined into `-b1e18i1`. +#### Symlink shortcuts +It's possible to invoke `zstd` through a symlink. +When the name of the symlink has a specific value, it triggers an associated behavior. +- `zstdmt` : compress using all cores available on local system. +- `zcat` : will decompress and output target file using any of the supported formats. `gzcat` and `zstdcat` are also equivalent. +- `gzip` : if zlib support is enabled, will mimic `gzip` by compressing file using `.gz` format, removing source file by default (use `--keep` to preserve). If zlib is not supported, triggers an error. +- `xz` : if lzma support is enabled, will mimic `xz` by compressing file using `.xz` format, removing source file by default (use `--keep` to preserve). If xz is not supported, triggers an error. +- `lzma` : if lzma support is enabled, will mimic `lzma` by compressing file using `.lzma` format, removing source file by default (use `--keep` to preserve). If lzma is not supported, triggers an error. +- `lz4` : if lz4 support is enabled, will mimic `lz4` by compressing file using `.lz4` format. If lz4 is not supported, triggers an error. +- `unzstd` and `unlz4` will decompress any of the supported format. +- `ungz`, `unxz` and `unlzma` will do the same, and will also remove source file by default (use `--keep` to preserve). + + #### Dictionary builder in Command Line Interface Zstd offers a training mode, which can be used to tune the algorithm for a selected type of data, by providing it with a few samples. The result of the training is stored @@ -107,7 +120,7 @@ Usage : FILE : a filename with no FILE, or when FILE is - , read standard input Arguments : - -# : # compression level (1-19, default:3) + -# : # compression level (1-19, default: 3) -d : decompression -D file: use `file` as Dictionary -o file: result stored into `file` (only if 1 input file) @@ -125,13 +138,13 @@ Advanced arguments : --ultra : enable levels beyond 19, up to 22 (requires more memory) --long : enable long distance matching (requires more memory) --no-dictID : don't write dictID into header (dictionary compression) ---[no-]check : integrity check (default:enabled) +--[no-]check : integrity check (default: enabled) -r : operate recursively on directories --format=gzip : compress files to the .gz format --format=xz : compress files to the .xz format --format=lzma : compress files to the .lzma format --test : test compressed file integrity ---[no-]sparse : sparse mode (default:disabled) +--[no-]sparse : sparse mode (default: disabled) -M# : Set a memory usage limit for decompression -- : All arguments after "--" are treated as files @@ -140,13 +153,13 @@ Dictionary builder : --train-cover[=k=#,d=#,steps=#] : use the cover algorithm with optional args --train-legacy[=s=#] : use the legacy algorithm with selectivity (default: 9) -o file : `file` is dictionary name (default: dictionary) ---maxdict=# : limit dictionary to specified size (default : 112640) +--maxdict=# : limit dictionary to specified size (default: 112640) --dictID=# : force dictionary ID to specified value (default: random) Benchmark arguments : - -b# : benchmark file(s), using # compression level (default : 1) + -b# : benchmark file(s), using # compression level (default: 3) -e# : test all compression levels from -bX to # (default: 1) - -i# : minimum evaluation time in seconds (default : 3s) + -i# : minimum evaluation time in seconds (default: 3s) -B# : cut file into independent blocks of size # (default: no block) --priority=rt : set process priority to real-time ``` @@ -155,7 +168,7 @@ Benchmark arguments : #### Long distance matching mode The long distance matching mode, enabled with `--long`, is designed to improve the compression ratio for files with long matches at a large distance (up to the -maximum window size, `128 MiB`) while still maintaining compression speed. +maximum window size, `128 MiB`) while still maintaining compression speed. Enabling this mode sets the window size to `128 MiB` and thus increases the memory usage for both the compressor and decompressor. Performance in terms of speed is @@ -168,7 +181,7 @@ decompression speed with and without long distance matching on an ideal use case: a tar of four versions of clang (versions `3.4.1`, `3.4.2`, `3.5.0`, `3.5.1`) with a total size of `244889600 B`. This is an ideal use case as there are many long distance matches within the maximum window size of `128 MiB` (each -version is less than `128 MiB`). +version is less than `128 MiB`). Compression Speed vs Ratio | Decompression Speed ---------------------------|--------------------- @@ -202,8 +215,3 @@ The below table illustrates this on the [Silesia compression corpus]. | `zstd -5 --long` | `3.319` | `51.7 MB/s` | `371.9 MB/s` | | `zstd -10` | `3.523` | `16.4 MB/s` | `489.2 MB/s` | | `zstd -10 --long`| `3.566` | `16.2 MB/s` | `415.7 MB/s` | - - - - - diff --git a/programs/bench.c b/programs/bench.c index 5d2568f104f50..014a4fd41b1cf 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -22,7 +22,7 @@ * Compiler Warnings ****************************************/ #ifdef _MSC_VER -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ #endif @@ -34,6 +34,7 @@ #include <stdlib.h> /* malloc, free */ #include <string.h> /* memset */ #include <stdio.h> /* fprintf, fopen */ +#include <assert.h> /* assert */ #include "mem.h" #define ZSTD_STATIC_LINKING_ONLY @@ -51,8 +52,9 @@ # define ZSTD_GIT_COMMIT_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_GIT_COMMIT) #endif -#define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */ -#define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */ +#define TIMELOOP_MICROSEC (1*1000000ULL) /* 1 second */ +#define TIMELOOP_NANOSEC (1*1000000000ULL) /* 1 second */ +#define ACTIVEPERIOD_MICROSEC (70*TIMELOOP_MICROSEC) /* 70 seconds */ #define COOLPERIOD_SEC 10 #define KB *(1 <<10) @@ -122,12 +124,12 @@ void BMK_setBlockSize(size_t blockSize) void BMK_setDecodeOnlyMode(unsigned decodeFlag) { g_decodeOnly = (decodeFlag>0); } -static U32 g_nbThreads = 1; -void BMK_setNbThreads(unsigned nbThreads) { +static U32 g_nbWorkers = 0; +void BMK_setNbWorkers(unsigned nbWorkers) { #ifndef ZSTD_MULTITHREAD - if (nbThreads > 1) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n"); + if (nbWorkers > 0) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n"); #endif - g_nbThreads = nbThreads; + g_nbWorkers = nbWorkers; } static U32 g_realTime = 0; @@ -212,6 +214,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, /* init */ if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* display last 17 characters */ + if (g_nbWorkers==1) g_nbWorkers=0; /* prefer synchronous mode */ if (g_decodeOnly) { /* benchmark only decompression : source must be already compressed */ const char* srcPtr = (const char*)srcBuffer; @@ -258,13 +261,19 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, } } } /* warmimg up memory */ - RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1); + if (g_decodeOnly) { + memcpy(compressedBuffer, srcBuffer, loadedCompressedSize); + } else { + RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1); + } /* Bench */ { U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL); U64 const crcOrig = g_decodeOnly ? 0 : XXH64(srcBuffer, srcSize, 0); UTIL_time_t coolTime; - U64 const maxTime = (g_nbSeconds * TIMELOOP_MICROSEC) + 1; + U64 const maxTime = (g_nbSeconds * TIMELOOP_NANOSEC) + 1; + U32 nbDecodeLoops = (U32)((100 MB) / (srcSize+1)) + 1; /* initial conservative speed estimate */ + U32 nbCompressionLoops = (U32)((2 MB) / (srcSize+1)) + 1; /* initial conservative speed estimate */ U64 totalCTime=0, totalDTime=0; U32 cCompleted=g_decodeOnly, dCompleted=0; # define NB_MARKS 4 @@ -283,19 +292,17 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, } if (!g_decodeOnly) { - UTIL_time_t clockStart; /* Compression */ DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize); if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */ - UTIL_sleepMilli(1); /* give processor time to other processes */ + UTIL_sleepMilli(5); /* give processor time to other processes */ UTIL_waitForNextTick(); - clockStart = UTIL_getTime(); if (!cCompleted) { /* still some time to do compression tests */ - U64 const clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1; U32 nbLoops = 0; - ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbThreads, g_nbThreads); + UTIL_time_t const clockStart = UTIL_getTime(); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, g_nbWorkers); ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel); ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag); ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmMinMatch, g_ldmMinMatch); @@ -307,13 +314,16 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashEveryLog, g_ldmHashEveryLog); } ZSTD_CCtx_setParameter(ctx, ZSTD_p_windowLog, comprParams->windowLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_hashLog, comprParams->hashLog); ZSTD_CCtx_setParameter(ctx, ZSTD_p_chainLog, comprParams->chainLog); ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog); ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength); ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength); ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy); ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); - do { + + if (!g_nbSeconds) nbCompressionLoops=1; + for (nbLoops=0; nbLoops<nbCompressionLoops; nbLoops++) { U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) { #if 0 /* direct compression function, for occasional comparison */ @@ -342,12 +352,16 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, } blockTable[blockNb].cSize = out.pos; #endif + } } + { U64 const loopDuration = UTIL_clockSpanNano(clockStart); + if (loopDuration > 0) { + if (loopDuration < fastestC * nbCompressionLoops) + fastestC = loopDuration / nbCompressionLoops; + nbCompressionLoops = (U32)(TIMELOOP_NANOSEC / fastestC) + 1; + } else { + assert(nbCompressionLoops < 40000000); /* avoid overflow */ + nbCompressionLoops *= 100; } - nbLoops++; - } while (UTIL_clockSpanMicro(clockStart) < clockLoop); - { U64 const loopDuration = UTIL_clockSpanMicro(clockStart); - if (loopDuration < fastestC*nbLoops) - fastestC = loopDuration / nbLoops; totalCTime += loopDuration; cCompleted = (totalCTime >= maxTime); /* end compression tests */ } } @@ -357,16 +371,14 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, ratio = (double)srcSize / (double)cSize; markNb = (markNb+1) % NB_MARKS; { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = (double)srcSize / fastestC; + double const compressionSpeed = ((double)srcSize / fastestC) * 1000; int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r", marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratioAccuracy, ratio, cSpeedAccuracy, compressionSpeed ); } - } else { /* g_decodeOnly */ - memcpy(compressedBuffer, srcBuffer, loadedCompressedSize); - } + } /* if (!g_decodeOnly) */ #if 0 /* disable decompression test */ dCompleted=1; @@ -375,16 +387,16 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, /* Decompression */ if (!dCompleted) memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */ - UTIL_sleepMilli(1); /* give processor time to other processes */ + UTIL_sleepMilli(5); /* give processor time to other processes */ UTIL_waitForNextTick(); if (!dCompleted) { - U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1; U32 nbLoops = 0; ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictBufferSize); UTIL_time_t const clockStart = UTIL_getTime(); if (!ddict) EXM_THROW(2, "ZSTD_createDDict() allocation failure"); - do { + if (!g_nbSeconds) nbDecodeLoops = 1; + for (nbLoops=0; nbLoops < nbDecodeLoops; nbLoops++) { U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) { size_t const regenSize = ZSTD_decompress_usingDDict(dctx, @@ -396,22 +408,26 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, blockNb, (U32)blockTable[blockNb].cSize, ZSTD_getErrorName(regenSize)); } blockTable[blockNb].resSize = regenSize; - } - nbLoops++; - } while (UTIL_clockSpanMicro(clockStart) < clockLoop); + } } ZSTD_freeDDict(ddict); - { U64 const loopDuration = UTIL_clockSpanMicro(clockStart); - if (loopDuration < fastestD*nbLoops) - fastestD = loopDuration / nbLoops; + { U64 const loopDuration = UTIL_clockSpanNano(clockStart); + if (loopDuration > 0) { + if (loopDuration < fastestD * nbDecodeLoops) + fastestD = loopDuration / nbDecodeLoops; + nbDecodeLoops = (U32)(TIMELOOP_NANOSEC / fastestD) + 1; + } else { + assert(nbDecodeLoops < 40000000); /* avoid overflow */ + nbDecodeLoops *= 100; + } totalDTime += loopDuration; dCompleted = (totalDTime >= maxTime); } } markNb = (markNb+1) % NB_MARKS; { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = (double)srcSize / fastestC; + double const compressionSpeed = ((double)srcSize / fastestC) * 1000; int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; - double const decompressionSpeed = (double)srcSize / fastestD; + double const decompressionSpeed = ((double)srcSize / fastestD) * 1000; DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratioAccuracy, ratio, @@ -460,8 +476,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, } /* for (testNb = 1; testNb <= (g_nbSeconds + !g_nbSeconds); testNb++) */ if (g_displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */ - double cSpeed = (double)srcSize / fastestC; - double dSpeed = (double)srcSize / fastestD; + double const cSpeed = ((double)srcSize / fastestC) * 1000; + double const dSpeed = ((double)srcSize / fastestD) * 1000; if (g_additionalParam) DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam); else @@ -518,9 +534,8 @@ static void BMK_benchCLevel(const void* srcBuffer, size_t benchedSize, if (g_displayLevel == 1 && !g_additionalParam) DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10)); - if (cLevelLast < cLevel) cLevelLast = cLevel; - for (l=cLevel; l <= cLevelLast; l++) { + if (l==0) continue; /* skip level 0 */ BMK_benchMem(srcBuffer, benchedSize, displayName, l, fileSizes, nbFiles, @@ -530,8 +545,8 @@ static void BMK_benchCLevel(const void* srcBuffer, size_t benchedSize, /*! BMK_loadFiles() : - Loads `buffer` with content of files listed within `fileNamesTable`. - At most, fills `buffer` entirely */ + * Loads `buffer` with content of files listed within `fileNamesTable`. + * At most, fills `buffer` entirely. */ static void BMK_loadFiles(void* buffer, size_t bufferSize, size_t* fileSizes, const char* const * const fileNamesTable, unsigned nbFiles) @@ -633,7 +648,8 @@ static void BMK_benchFileTable(const char* const * const fileNamesTable, unsigne } -static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, const ZSTD_compressionParameters* compressionParams) +static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, + const ZSTD_compressionParameters* compressionParams) { char name[20] = {0}; size_t benchedSize = 10000000; @@ -661,7 +677,6 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, { double const compressibility = (double)g_compressibilityDefault / 100; - if (cLevel < 1) cLevel = 1; /* minimum compression level */ if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel(); if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel(); if (cLevelLast < cLevel) cLevelLast = cLevel; diff --git a/programs/bench.h b/programs/bench.h index d2ec94ee36888..bf1087013feb1 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -22,7 +22,7 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* di /* Set Parameters */ void BMK_setNbSeconds(unsigned nbLoops); void BMK_setBlockSize(size_t blockSize); -void BMK_setNbThreads(unsigned nbThreads); +void BMK_setNbWorkers(unsigned nbWorkers); void BMK_setRealTime(unsigned priority); void BMK_setNotificationLevel(unsigned level); void BMK_setSeparateFiles(unsigned separate); diff --git a/programs/fileio.c b/programs/fileio.c index b57ce14d2f322..14569bb47528f 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -36,18 +36,21 @@ # include <io.h> #endif -#include "bitstream.h" #include "mem.h" #include "fileio.h" #include "util.h" + #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ #include "zstd.h" +#include "zstd_errors.h" /* ZSTD_error_frameParameter_windowTooLarge */ + #if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS) # include <zlib.h> # if !defined(z_const) # define z_const # endif #endif + #if defined(ZSTD_LZMACOMPRESS) || defined(ZSTD_LZMADECOMPRESS) # include <lzma.h> #endif @@ -84,10 +87,15 @@ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; } static const U64 g_refreshRate = SEC_TO_MICRO / 6; static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; -#define DISPLAYUPDATE(l, ...) { if (g_displayLevel>=l) { \ - if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \ - { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stderr); } } } +#define READY_FOR_UPDATE() (UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) +#define DELAY_NEXT_UPDATE() { g_displayClock = UTIL_getTime(); } +#define DISPLAYUPDATE(l, ...) { \ + if (g_displayLevel>=l) { \ + if (READY_FOR_UPDATE() || (g_displayLevel>=4)) { \ + DELAY_NEXT_UPDATE(); \ + DISPLAY(__VA_ARGS__); \ + if (g_displayLevel>=4) fflush(stderr); \ + } } } #undef MIN /* in case it would be already defined */ #define MIN(a,b) ((a) < (b) ? (a) : (b)) @@ -139,7 +147,10 @@ static void INThandler(int sig) #if !defined(_MSC_VER) signal(sig, SIG_IGN); /* this invocation generates a buggy warning in Visual Studio */ #endif - if (g_artefact) remove(g_artefact); + if (g_artefact) { + assert(UTIL_isRegularFile(g_artefact)); + remove(g_artefact); + } DISPLAY("\n"); exit(2); } @@ -209,23 +220,23 @@ static U32 g_removeSrcFile = 0; void FIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); } static U32 g_memLimit = 0; void FIO_setMemLimit(unsigned memLimit) { g_memLimit = memLimit; } -static U32 g_nbThreads = 1; -void FIO_setNbThreads(unsigned nbThreads) { +static U32 g_nbWorkers = 1; +void FIO_setNbWorkers(unsigned nbWorkers) { #ifndef ZSTD_MULTITHREAD - if (nbThreads > 1) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n"); + if (nbWorkers > 0) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n"); #endif - g_nbThreads = nbThreads; + g_nbWorkers = nbWorkers; } static U32 g_blockSize = 0; void FIO_setBlockSize(unsigned blockSize) { - if (blockSize && g_nbThreads==1) + if (blockSize && g_nbWorkers==0) DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n"); g_blockSize = blockSize; } #define FIO_OVERLAP_LOG_NOTSET 9999 static U32 g_overlapLog = FIO_OVERLAP_LOG_NOTSET; void FIO_setOverlapLog(unsigned overlapLog){ - if (overlapLog && g_nbThreads==1) + if (overlapLog && g_nbWorkers==0) DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n"); g_overlapLog = overlapLog; } @@ -392,9 +403,9 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName) #ifndef ZSTD_NOCOMPRESS -/*-********************************************************************** -* Compression -************************************************************************/ +/* ********************************************************************** + * Compression + ************************************************************************/ typedef struct { FILE* srcFile; FILE* dstFile; @@ -411,6 +422,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, cRess_t ress; memset(&ress, 0, sizeof(ress)); + DISPLAYLEVEL(6, "FIO_createCResources \n"); ress.cctx = ZSTD_createCCtx(); if (ress.cctx == NULL) EXM_THROW(30, "allocation error : can't create ZSTD_CCtx"); @@ -421,7 +433,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "allocation error : not enough memory"); - /* Advances parameters, including dictionary */ + /* Advanced parameters, including dictionary */ { void* dictBuffer; size_t const dictBuffSize = FIO_createDictBuffer(&dictBuffer, dictFileName); /* works with dictFileName==NULL */ if (dictFileName && (dictBuffer==NULL)) @@ -431,10 +443,9 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) ); /* compression level */ - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, cLevel) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)cLevel) ); /* long distance matching */ - CHECK( ZSTD_CCtx_setParameter( - ress.cctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashLog, g_ldmHashLog) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmMinMatch, g_ldmMinMatch) ); if (g_ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) { @@ -452,10 +463,12 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_targetLength, comprParams->targetLength) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionStrategy, (U32)comprParams->strategy) ); /* multi-threading */ - DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) ); +#ifdef ZSTD_MULTITHREAD + DISPLAYLEVEL(5,"set nb workers = %u \n", g_nbWorkers); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbWorkers, g_nbWorkers) ); +#endif /* dictionary */ - CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* just for dictionary loading, for compression parameters adaptation */ + CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* set the value temporarily for dictionary loading, to adapt compression parameters */ CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) ); CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reset */ @@ -726,19 +739,93 @@ static unsigned long long FIO_compressLz4Frame(cRess_t* ress, * @return : 0 : compression completed correctly, * 1 : missing or pb opening srcFileName */ -static int FIO_compressFilename_internal(cRess_t ress, - const char* dstFileName, const char* srcFileName, int compressionLevel) +static unsigned long long +FIO_compressZstdFrame(const cRess_t* ressPtr, + const char* srcFileName, U64 fileSize, + int compressionLevel, U64* readsize) { + cRess_t const ress = *ressPtr; FILE* const srcFile = ress.srcFile; FILE* const dstFile = ress.dstFile; + U64 compressedfilesize = 0; + ZSTD_EndDirective directive = ZSTD_e_continue; + DISPLAYLEVEL(6, "compression using zstd format \n"); + + /* init */ + if (fileSize != UTIL_FILESIZE_UNKNOWN) + ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize); + (void)compressionLevel; (void)srcFileName; + + /* Main compression loop */ + do { + size_t result; + /* Fill input Buffer */ + size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile); + ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 }; + DISPLAYLEVEL(6, "fread %u bytes from source \n", (U32)inSize); + *readsize += inSize; + + if ((inSize == 0) || (*readsize == fileSize)) + directive = ZSTD_e_end; + + result = 1; + while (inBuff.pos != inBuff.size || (directive == ZSTD_e_end && result != 0)) { + ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 }; + CHECK_V(result, ZSTD_compress_generic(ress.cctx, &outBuff, &inBuff, directive)); + + /* Write compressed stream */ + DISPLAYLEVEL(6, "ZSTD_compress_generic(end:%u) => intput pos(%u)<=(%u)size ; output generated %u bytes \n", + (U32)directive, (U32)inBuff.pos, (U32)inBuff.size, (U32)outBuff.pos); + if (outBuff.pos) { + size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile); + if (sizeCheck!=outBuff.pos) + EXM_THROW(25, "Write error : cannot write compressed block"); + compressedfilesize += outBuff.pos; + } + if (READY_FOR_UPDATE()) { + ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress.cctx); + double const cShare = (double)zfp.produced / (zfp.consumed + !zfp.consumed/*avoid div0*/) * 100; + if (g_displayLevel >= 3) { + DISPLAYUPDATE(3, "\r(L%i) Buffered :%4u MB - Consumed :%4u MB - Compressed :%4u MB => %.2f%%", + compressionLevel, + (U32)((zfp.ingested - zfp.consumed) >> 20), + (U32)(zfp.consumed >> 20), + (U32)(zfp.produced >> 20), + cShare ); + } else { /* g_displayLevel == 2 */ + DISPLAYLEVEL(2, "\rRead : %u ", (U32)(zfp.consumed >> 20)); + if (fileSize != UTIL_FILESIZE_UNKNOWN) + DISPLAYLEVEL(2, "/ %u ", (U32)(fileSize >> 20)); + DISPLAYLEVEL(2, "MB ==> %2.f%% ", cShare); + DELAY_NEXT_UPDATE(); + } + } + } + } while (directive != ZSTD_e_end); + + return compressedfilesize; +} + +/*! FIO_compressFilename_internal() : + * same as FIO_compressFilename_extRess(), with `ress.desFile` already opened. + * @return : 0 : compression completed correctly, + * 1 : missing or pb opening srcFileName + */ +static int +FIO_compressFilename_internal(cRess_t ress, + const char* dstFileName, const char* srcFileName, + int compressionLevel) +{ U64 readsize = 0; U64 compressedfilesize = 0; U64 const fileSize = UTIL_getFileSize(srcFileName); - ZSTD_EndDirective directive = ZSTD_e_continue; DISPLAYLEVEL(5, "%s: %u bytes \n", srcFileName, (U32)fileSize); + /* compression format selection */ switch (g_compressionType) { + default: case FIO_zstdCompression: + compressedfilesize = FIO_compressZstdFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize); break; case FIO_gzipCompression: @@ -749,7 +836,7 @@ static int FIO_compressFilename_internal(cRess_t ress, EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n", srcFileName); #endif - goto finish; + break; case FIO_xzCompression: case FIO_lzmaCompression: @@ -760,7 +847,7 @@ static int FIO_compressFilename_internal(cRess_t ress, EXM_THROW(20, "zstd: %s: file cannot be compressed as xz/lzma (zstd compiled without ZSTD_LZMACOMPRESS) -- ignored \n", srcFileName); #endif - goto finish; + break; case FIO_lz4Compression: #ifdef ZSTD_LZ4COMPRESS @@ -770,62 +857,14 @@ static int FIO_compressFilename_internal(cRess_t ress, EXM_THROW(20, "zstd: %s: file cannot be compressed as lz4 (zstd compiled without ZSTD_LZ4COMPRESS) -- ignored \n", srcFileName); #endif - goto finish; + break; } - /* init */ - if (fileSize != UTIL_FILESIZE_UNKNOWN) - ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize); - - /* Main compression loop */ - do { - size_t result; - /* Fill input Buffer */ - size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile); - ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 }; - readsize += inSize; - - if (inSize == 0 || (fileSize != UTIL_FILESIZE_UNKNOWN && readsize == fileSize)) - directive = ZSTD_e_end; - - result = 1; - while (inBuff.pos != inBuff.size || (directive == ZSTD_e_end && result != 0)) { - ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 }; - CHECK_V(result, ZSTD_compress_generic(ress.cctx, &outBuff, &inBuff, directive)); - - /* Write compressed stream */ - DISPLAYLEVEL(6, "ZSTD_compress_generic,ZSTD_e_continue: generated %u bytes \n", - (U32)outBuff.pos); - if (outBuff.pos) { - size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile); - if (sizeCheck!=outBuff.pos) - EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName); - compressedfilesize += outBuff.pos; - } - } - if (g_nbThreads > 1) { - if (fileSize == UTIL_FILESIZE_UNKNOWN) - DISPLAYUPDATE(2, "\rRead : %u MB", (U32)(readsize>>20)) - else - DISPLAYUPDATE(2, "\rRead : %u / %u MB", - (U32)(readsize>>20), (U32)(fileSize>>20)); - } else { - if (fileSize == UTIL_FILESIZE_UNKNOWN) - DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", - (U32)(readsize>>20), - (double)compressedfilesize/readsize*100) - else - DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", - (U32)(readsize>>20), (U32)(fileSize>>20), - (double)compressedfilesize/readsize*100); - } - } while (directive != ZSTD_e_end); - -finish: /* Status */ DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", srcFileName, - (double)compressedfilesize/(readsize+(!readsize) /* avoid div by zero */ )*100, + DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", + srcFileName, + (double)compressedfilesize / (readsize+(!readsize)/*avoid div by zero*/) * 100, (unsigned long long)readsize, (unsigned long long) compressedfilesize, dstFileName); @@ -861,7 +900,7 @@ static int FIO_compressFilename_srcFile(cRess_t ress, * delete both the source and destination files. */ clearHandler(); - if (remove(srcFileName)) + if (FIO_remove(srcFileName)) EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno)); } return result; @@ -881,6 +920,7 @@ static int FIO_compressFilename_dstFile(cRess_t ress, stat_t statbuf; int stat_result = 0; + DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s", dstFileName); ress.dstFile = FIO_openDstFile(dstFileName); if (ress.dstFile==NULL) return 1; /* could not open dstFileName */ /* Must ony be added after FIO_openDstFile() succeeds. @@ -898,11 +938,13 @@ static int FIO_compressFilename_dstFile(cRess_t ress, DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result=1; } - if (result!=0) { /* remove operation artefact */ - if (remove(dstFileName)) - EXM_THROW(1, "zstd: %s: %s", dstFileName, strerror(errno)); - } - else if (strcmp (dstFileName, stdoutmark) && stat_result) + if ( (result != 0) /* operation failure */ + && strcmp(dstFileName, nulmark) /* special case : don't remove() /dev/null */ + && strcmp(dstFileName, stdoutmark) ) /* special case : don't remove() stdout */ + FIO_remove(dstFileName); /* remove compression artefact; note don't do anything special if remove() fails */ + else if ( strcmp(dstFileName, stdoutmark) + && strcmp(dstFileName, nulmark) + && stat_result) UTIL_setFileStat(dstFileName, &statbuf); return result; @@ -951,10 +993,14 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile if (outFileName != NULL) { unsigned u; ress.dstFile = FIO_openDstFile(outFileName); - for (u=0; u<nbFiles; u++) - missed_files += FIO_compressFilename_srcFile(ress, outFileName, inFileNamesTable[u], compressionLevel); - if (fclose(ress.dstFile)) - EXM_THROW(29, "Write error : cannot properly close stdout"); + if (ress.dstFile==NULL) { /* could not open outFileName */ + missed_files = nbFiles; + } else { + for (u=0; u<nbFiles; u++) + missed_files += FIO_compressFilename_srcFile(ress, outFileName, inFileNamesTable[u], compressionLevel); + if (fclose(ress.dstFile)) + EXM_THROW(29, "Write error : cannot properly close stdout"); + } } else { unsigned u; for (u=0; u<nbFiles; u++) { @@ -1134,33 +1180,46 @@ static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_ return 0; } -static void FIO_zstdErrorHelp(dRess_t* ress, size_t ret, char const* srcFileName) +/* FIO_highbit64() : + * gives position of highest bit. + * note : only works for v > 0 ! + */ +static unsigned FIO_highbit64(unsigned long long v) +{ + unsigned count = 0; + assert(v != 0); + v >>= 1; + while (v) { v >>= 1; count++; } + return count; +} + +/* FIO_zstdErrorHelp() : + * detailed error message when requested window size is too large */ +static void FIO_zstdErrorHelp(dRess_t* ress, size_t err, char const* srcFileName) { ZSTD_frameHeader header; - /* No special help for these errors */ - if (ZSTD_getErrorCode(ret) != ZSTD_error_frameParameter_windowTooLarge) + + /* Help message only for one specific error */ + if (ZSTD_getErrorCode(err) != ZSTD_error_frameParameter_windowTooLarge) return; + /* Try to decode the frame header */ - ret = ZSTD_getFrameHeader(&header, ress->srcBuffer, ress->srcBufferLoaded); - if (ret == 0) { - U32 const windowSize = (U32)header.windowSize; - U32 const windowLog = BIT_highbit32(windowSize) + ((windowSize & (windowSize - 1)) != 0); - U32 const windowMB = (windowSize >> 20) + ((windowSize & ((1 MB) - 1)) != 0); - assert(header.windowSize <= (U64)((U32)-1)); + err = ZSTD_getFrameHeader(&header, ress->srcBuffer, ress->srcBufferLoaded); + if (err == 0) { + unsigned long long const windowSize = header.windowSize; + U32 const windowLog = FIO_highbit64(windowSize) + ((windowSize & (windowSize - 1)) != 0); + U32 const windowMB = (U32)((windowSize >> 20) + ((windowSize & ((1 MB) - 1)) != 0)); + assert(windowSize < (U64)(1ULL << 52)); assert(g_memLimit > 0); DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u\n", - srcFileName, header.windowSize, g_memLimit); + srcFileName, windowSize, g_memLimit); if (windowLog <= ZSTD_WINDOWLOG_MAX) { DISPLAYLEVEL(1, "%s : Use --long=%u or --memory=%uMB\n", srcFileName, windowLog, windowMB); return; } - } else if (ZSTD_getErrorCode(ret) != ZSTD_error_frameParameter_windowTooLarge) { - DISPLAYLEVEL(1, "%s : Error decoding frame header to read window size : %s\n", - srcFileName, ZSTD_getErrorName(ret)); - return; } - DISPLAYLEVEL(1, "%s : Window log larger than ZSTD_WINDOWLOG_MAX=%u not supported\n", + DISPLAYLEVEL(1, "%s : Window log larger than ZSTD_WINDOWLOG_MAX=%u; not supported\n", srcFileName, ZSTD_WINDOWLOG_MAX); } @@ -1571,7 +1630,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch * delete both the source and destination files. */ clearHandler(); - if (remove(srcFileName)) { + if (FIO_remove(srcFileName)) { /* failed to remove src file */ DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); return 1; @@ -1614,7 +1673,7 @@ static int FIO_decompressDstFile(dRess_t ress, if ( (result != 0) /* operation failure */ && strcmp(dstFileName, nulmark) /* special case : don't remove() /dev/null (#316) */ && strcmp(dstFileName, stdoutmark) ) /* special case : don't remove() stdout */ - remove(dstFileName); /* remove decompression artefact; note don't do anything special if remove() fails */ + FIO_remove(dstFileName); /* remove decompression artefact; note don't do anything special if remove() fails */ else { /* operation success */ if ( strcmp(dstFileName, stdoutmark) /* special case : don't chmod stdout */ && strcmp(dstFileName, nulmark) /* special case : don't chmod /dev/null */ @@ -1917,6 +1976,7 @@ static void displayInfo(const char* inFileName, const fileInfo_t* info, int disp static fileInfo_t FIO_addFInfo(fileInfo_t fi1, fileInfo_t fi2) { fileInfo_t total; + memset(&total, 0, sizeof(total)); total.numActualFrames = fi1.numActualFrames + fi2.numActualFrames; total.numSkippableFrames = fi1.numSkippableFrames + fi2.numSkippableFrames; total.compressedSize = fi1.compressedSize + fi2.compressedSize; diff --git a/programs/fileio.h b/programs/fileio.h index 9b9c7ea2fadce..69c83f71dce3d 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -54,7 +54,7 @@ void FIO_setDictIDFlag(unsigned dictIDFlag); void FIO_setChecksumFlag(unsigned checksumFlag); void FIO_setRemoveSrcFile(unsigned flag); void FIO_setMemLimit(unsigned memLimit); -void FIO_setNbThreads(unsigned nbThreads); +void FIO_setNbWorkers(unsigned nbWorkers); void FIO_setBlockSize(unsigned blockSize); void FIO_setOverlapLog(unsigned overlapLog); void FIO_setLdmFlag(unsigned ldmFlag); diff --git a/programs/platform.h b/programs/platform.h index 81dee23f91731..c86d289f5414e 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -71,7 +71,7 @@ extern "C" { ***************************************************************/ #if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) /* UNIX-like OS */ \ || defined(__midipix__) || defined(__VMS)) -# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1–2001 (SUSv3) conformant */ \ +# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1-2001 (SUSv3) conformant */ \ || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* BSD distros */ # define PLATFORM_POSIX_VERSION 200112L # else @@ -106,8 +106,7 @@ extern "C" { # include <io.h> /* _isatty */ # include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ # include <stdio.h> /* FILE */ -static __inline int IS_CONSOLE(FILE* stdStream) -{ +static __inline int IS_CONSOLE(FILE* stdStream) { DWORD dummy; return _isatty(_fileno(stdStream)) && GetConsoleMode((HANDLE)_get_osfhandle(_fileno(stdStream)), &dummy); } diff --git a/programs/util.h b/programs/util.h index af1fa7fcae749..3e69745793e4e 100644 --- a/programs/util.h +++ b/programs/util.h @@ -142,7 +142,9 @@ static int g_utilDisplayLevel; } return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; } + #elif defined(__APPLE__) && defined(__MACH__) + #include <mach/mach_time.h> #define UTIL_TIME_INITIALIZER 0 typedef U64 UTIL_time_t; @@ -167,7 +169,9 @@ static int g_utilDisplayLevel; } return ((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom); } + #elif (PLATFORM_POSIX_VERSION >= 200112L) && (defined __UCLIBC__ || ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 17) || __GLIBC__ > 2)) + #define UTIL_TIME_INITIALIZER { 0, 0 } typedef struct timespec UTIL_freq_t; typedef struct timespec UTIL_time_t; @@ -217,12 +221,18 @@ static int g_utilDisplayLevel; #define SEC_TO_MICRO 1000000 /* returns time span in microseconds */ -UTIL_STATIC U64 UTIL_clockSpanMicro( UTIL_time_t clockStart ) +UTIL_STATIC U64 UTIL_clockSpanMicro(UTIL_time_t clockStart ) { UTIL_time_t const clockEnd = UTIL_getTime(); return UTIL_getSpanTimeMicro(clockStart, clockEnd); } +/* returns time span in microseconds */ +UTIL_STATIC U64 UTIL_clockSpanNano(UTIL_time_t clockStart ) +{ + UTIL_time_t const clockEnd = UTIL_getTime(); + return UTIL_getSpanTimeNano(clockStart, clockEnd); +} UTIL_STATIC void UTIL_waitForNextTick(void) { @@ -246,11 +256,17 @@ UTIL_STATIC void UTIL_waitForNextTick(void) #endif +UTIL_STATIC int UTIL_isRegularFile(const char* infilename); + + UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf) { int res = 0; struct utimbuf timebuf; + if (!UTIL_isRegularFile(filename)) + return -1; + timebuf.actime = time(NULL); timebuf.modtime = statbuf->st_mtime; res += utime(filename, &timebuf); /* set access and modification times */ diff --git a/programs/zstd.1 b/programs/zstd.1 index e0902e8582c60..8e9e83745a0f8 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -1,5 +1,5 @@ . -.TH "ZSTD" "1" "December 2017" "zstd 1.3.3" "User Commands" +.TH "ZSTD" "1" "2018-01-27" "zstd 1.3.4" "User Commands" . .SH "NAME" \fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files @@ -136,7 +136,7 @@ force write to standard output, even if it is the console . .TP \fB\-\-[no\-]sparse\fR -enable / disable sparse FS support, to make files with many zeroes smaller on disk\. Creating sparse files may save disk space and speed up decompression by reducing the amount of disk I/O\. default : enabled when output is into a file, and disabled when output is stdout\. This setting overrides default and can force sparse mode over stdout\. +enable / disable sparse FS support, to make files with many zeroes smaller on disk\. Creating sparse files may save disk space and speed up decompression by reducing the amount of disk I/O\. default: enabled when output is into a file, and disabled when output is stdout\. This setting overrides default and can force sparse mode over stdout\. . .TP \fB\-\-rm\fR @@ -172,14 +172,14 @@ suppress warnings, interactivity, and notifications\. specify twice to suppress . .TP \fB\-C\fR, \fB\-\-[no\-]check\fR -add integrity check computed from uncompressed data (default : enabled) +add integrity check computed from uncompressed data (default: enabled) . .TP \fB\-\-\fR All arguments after \fB\-\-\fR are treated as files . .SH "DICTIONARY BUILDER" -\fBzstd\fR offers \fIdictionary\fR compression, useful for very small files and messages\. It\'s possible to train \fBzstd\fR with some samples, the result of which is saved into a file called a \fBdictionary\fR\. Then during compression and decompression, reference the same dictionary\. It will improve compression ratio of small files\. Typical gains range from 10% (at 64KB) to x5 better (at <1KB)\. +\fBzstd\fR offers \fIdictionary\fR compression, which greatly improves efficiency on small files and messages\. It\'s possible to train \fBzstd\fR with a set of samples, the result of which is saved into a file called a \fBdictionary\fR\. Then during compression and decompression, reference the same dictionary, using command \fB\-D dictionaryFileName\fR\. Compression of small files similar to the sample set will be greatly improved\. . .TP \fB\-\-train FILEs\fR @@ -197,6 +197,10 @@ Dictionary saved into \fBfile\fR (default name: dictionary)\. Limit dictionary to specified size (default: 112640)\. . .TP +\fB\-#\fR +Use \fB#\fR compression level during training (optional)\. Will generate statistics more tuned for selected compression level, resulting in a \fIsmall\fR compression ratio improvement for this level\. +. +.TP \fB\-B#\fR Split input files in blocks of size # (default: no split) . @@ -251,7 +255,7 @@ benchmark file(s) using multiple compression levels, from \fB\-b#\fR to \fB\-e#\ . .TP \fB\-i#\fR -minimum evaluation time, in seconds (default : 3s), benchmark mode only +minimum evaluation time, in seconds (default: 3s), benchmark mode only . .TP \fB\-B#\fR, \fB\-\-block\-size=#\fR diff --git a/programs/zstd.1.md b/programs/zstd.1.md index 9e4c5d305550a..2e2dc54f86685 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -115,11 +115,25 @@ the last one takes effect. Note: If `windowLog` is set to larger than 27, `--long=windowLog` or `--memory=windowSize` needs to be passed to the decompressor. +* `--fast[=#]`: + switch to ultra-fast compression levels. + If `=#` is not present, it defaults to `1`. + The higher the value, the faster the compression speed, + at the cost of some compression ratio. + This setting overwrites compression level if one was set previously. + Similarly, if a compression level is set after `--fast`, it overrides it. + * `-T#`, `--threads=#`: - Compress using `#` threads (default: 1). + Compress using `#` working threads (default: 1). If `#` is 0, attempt to detect and use the number of physical CPU cores. - In all cases, the nb of threads is capped to ZSTDMT_NBTHREADS_MAX==256. + In all cases, the nb of threads is capped to ZSTDMT_NBTHREADS_MAX==200. This modifier does nothing if `zstd` is compiled without multithread support. +* `--single-thread`: + Does not spawn a thread for compression, use caller thread instead. + This is the only available mode when multithread support is disabled. + In this mode, compression is serialized with I/O. + (This is different from `-T1`, which spawns 1 compression thread in parallel of I/O). + Single-thread mode also features lower memory usage. * `-D file`: use `file` as Dictionary to compress or decompress FILE(s) * `--nodictID`: @@ -137,7 +151,7 @@ the last one takes effect. to make files with many zeroes smaller on disk. Creating sparse files may save disk space and speed up decompression by reducing the amount of disk I/O. - default : enabled when output is into a file, + default: enabled when output is into a file, and disabled when output is stdout. This setting overrides default and can force sparse mode over stdout. * `--rm`: @@ -163,7 +177,7 @@ the last one takes effect. suppress warnings, interactivity, and notifications. specify twice to suppress errors too. * `-C`, `--[no-]check`: - add integrity check computed from uncompressed data (default : enabled) + add integrity check computed from uncompressed data (default: enabled) * `--`: All arguments after `--` are treated as files @@ -171,12 +185,12 @@ the last one takes effect. DICTIONARY BUILDER ------------------ `zstd` offers _dictionary_ compression, -useful for very small files and messages. -It's possible to train `zstd` with some samples, +which greatly improves efficiency on small files and messages. +It's possible to train `zstd` with a set of samples, the result of which is saved into a file called a `dictionary`. -Then during compression and decompression, reference the same dictionary. -It will improve compression ratio of small files. -Typical gains range from 10% (at 64KB) to x5 better (at <1KB). +Then during compression and decompression, reference the same dictionary, +using command `-D dictionaryFileName`. +Compression of small files similar to the sample set will be greatly improved. * `--train FILEs`: Use FILEs as training set to create a dictionary. @@ -192,6 +206,10 @@ Typical gains range from 10% (at 64KB) to x5 better (at <1KB). Dictionary saved into `file` (default name: dictionary). * `--maxdict=#`: Limit dictionary to specified size (default: 112640). +* `-#`: + Use `#` compression level during training (optional). + Will generate statistics more tuned for selected compression level, + resulting in a _small_ compression ratio improvement for this level. * `-B#`: Split input files in blocks of size # (default: no split) * `--dictID=#`: @@ -252,7 +270,7 @@ BENCHMARK * `-e#`: benchmark file(s) using multiple compression levels, from `-b#` to `-e#` (inclusive) * `-i#`: - minimum evaluation time, in seconds (default : 3s), benchmark mode only + minimum evaluation time, in seconds (default: 3s), benchmark mode only * `-B#`, `--block-size=#`: cut file(s) into independent blocks of size # (default: no block) * `--priority=rt`: @@ -329,14 +347,21 @@ The list of available _options_: The minimum _slen_ is 3 and the maximum is 7. - `targetLen`=_tlen_, `tlen`=_tlen_: - Specify the minimum match length that causes a match finder to stop - searching for better matches. + The impact of this field vary depending on selected strategy. + + For ZSTD\_btopt and ZSTD\_btultra, it specifies the minimum match length + that causes match finder to stop searching for better matches. + A larger `targetLen` usually improves compression ratio + but decreases compression speed. + + For ZSTD\_fast, it specifies + the amount of data skipped between match sampling. + Impact is reversed : a larger `targetLen` increases compression speed + but decreases compression ratio. - A larger minimum match length usually improves compression ratio but - decreases compression speed. - This option is only used with strategies ZSTD_btopt and ZSTD_btultra. + For all other strategies, this field has no impact. - The minimum _tlen_ is 4 and the maximum is 999. + The minimum _tlen_ is 1 and the maximum is 999. - `overlapLog`=_ovlog_, `ovlog`=_ovlog_: Determine `overlapSize`, amount of data reloaded from previous job. diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 7e29998b0f5e8..c35de7ccfbbd3 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -54,6 +54,7 @@ #define ZSTD_ZSTDMT "zstdmt" #define ZSTD_UNZSTD "unzstd" #define ZSTD_CAT "zstdcat" +#define ZSTD_ZCAT "zcat" #define ZSTD_GZ "gzip" #define ZSTD_GUNZIP "gunzip" #define ZSTD_GZCAT "gzcat" @@ -105,7 +106,7 @@ static int usage(const char* programName) DISPLAY( " with no FILE, or when FILE is - , read standard input\n"); DISPLAY( "Arguments : \n"); #ifndef ZSTD_NOCOMPRESS - DISPLAY( " -# : # compression level (1-%d, default:%d) \n", ZSTDCLI_CLEVEL_MAX, ZSTDCLI_CLEVEL_DEFAULT); + DISPLAY( " -# : # compression level (1-%d, default: %d) \n", ZSTDCLI_CLEVEL_MAX, ZSTDCLI_CLEVEL_DEFAULT); #endif #ifndef ZSTD_NODECOMPRESS DISPLAY( " -d : decompression \n"); @@ -132,13 +133,14 @@ static int usage_advanced(const char* programName) DISPLAY( " -l : print information about zstd compressed files \n"); #ifndef ZSTD_NOCOMPRESS DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel()); - DISPLAY( "--long[=#] : enable long distance matching with given window log (default : %u)\n", g_defaultMaxWindowLog); + DISPLAY( "--long[=#]: enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog); + DISPLAY( "--fast[=#]: switch to ultra fast compression level (default: %u)\n", 1); #ifdef ZSTD_MULTITHREAD - DISPLAY( " -T# : use # threads for compression (default:1) \n"); - DISPLAY( " -B# : select size of each job (default:0==automatic) \n"); + DISPLAY( " -T# : spawns # compression threads (default: 1, 0==# cores) \n"); + DISPLAY( " -B# : select size of each job (default: 0==automatic) \n"); #endif DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n"); - DISPLAY( "--[no-]check : integrity check (default:enabled) \n"); + DISPLAY( "--[no-]check : integrity check (default: enabled) \n"); #endif #ifdef UTIL_HAS_CREATEFILELIST DISPLAY( " -r : operate recursively on directories \n"); @@ -156,9 +158,9 @@ static int usage_advanced(const char* programName) #ifndef ZSTD_NODECOMPRESS DISPLAY( "--test : test compressed file integrity \n"); #if ZSTD_SPARSE_DEFAULT - DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); + DISPLAY( "--[no-]sparse : sparse mode (default: enabled on file, disabled on stdout)\n"); #else - DISPLAY( "--[no-]sparse : sparse mode (default:disabled)\n"); + DISPLAY( "--[no-]sparse : sparse mode (default: disabled)\n"); #endif #endif DISPLAY( " -M# : Set a memory usage limit for decompression \n"); @@ -170,15 +172,15 @@ static int usage_advanced(const char* programName) DISPLAY( "--train-cover[=k=#,d=#,steps=#] : use the cover algorithm with optional args\n"); DISPLAY( "--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: %u)\n", g_defaultSelectivityLevel); DISPLAY( " -o file : `file` is dictionary name (default: %s) \n", g_defaultDictName); - DISPLAY( "--maxdict=# : limit dictionary to specified size (default : %u) \n", g_defaultMaxDictSize); + DISPLAY( "--maxdict=# : limit dictionary to specified size (default: %u) \n", g_defaultMaxDictSize); DISPLAY( "--dictID=# : force dictionary ID to specified value (default: random)\n"); #endif #ifndef ZSTD_NOBENCH DISPLAY( "\n"); DISPLAY( "Benchmark arguments : \n"); - DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n"); + DISPLAY( " -b# : benchmark file(s), using # compression level (default: %d) \n", ZSTDCLI_CLEVEL_DEFAULT); DISPLAY( " -e# : test all compression levels from -bX to # (default: 1)\n"); - DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s) \n"); + DISPLAY( " -i# : minimum evaluation time in seconds (default: 3s) \n"); DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n"); DISPLAY( "--priority=rt : set process priority to real-time \n"); #endif @@ -218,10 +220,10 @@ static int exeNameMatch(const char* exeName, const char* test) } /*! readU32FromChar() : - @return : unsigned integer value read from input in `char` format - allows and interprets K, KB, KiB, M, MB and MiB suffix. - Will also modify `*stringPtr`, advancing it to position where it stopped reading. - Note : function result can overflow if digit string > MAX_UINT */ + * @return : unsigned integer value read from input in `char` format. + * allows and interprets K, KB, KiB, M, MB and MiB suffix. + * Will also modify `*stringPtr`, advancing it to position where it stopped reading. + * Note : function result can overflow if digit string > MAX_UINT */ static unsigned readU32FromChar(const char** stringPtr) { unsigned result = 0; @@ -240,7 +242,7 @@ static unsigned readU32FromChar(const char** stringPtr) /** longCommandWArg() : * check if *stringPtr is the same as longCommand. * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. - * @return 0 and doesn't modify *stringPtr otherwise. + * @return 0 and doesn't modify *stringPtr otherwise. */ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) { @@ -318,12 +320,13 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi if (longCommandWArg(&stringPtr, "ldmSearchLength=") || longCommandWArg(&stringPtr, "ldmslen=")) { g_ldmMinMatch = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "ldmBucketSizeLog=") || longCommandWArg(&stringPtr, "ldmblog=")) { g_ldmBucketSizeLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "ldmHashEveryLog=") || longCommandWArg(&stringPtr, "ldmhevery=")) { g_ldmHashEveryLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + DISPLAYLEVEL(4, "invalid compression parameter \n"); return 0; } + DISPLAYLEVEL(4, "windowLog=%d, chainLog=%d, hashLog=%d, searchLog=%d \n", params->windowLog, params->chainLog, params->hashLog, params->searchLog); + DISPLAYLEVEL(4, "searchLength=%d, targetLength=%d, strategy=%d \n", params->searchLength, params->targetLength, params->strategy); if (stringPtr[0] != 0) return 0; /* check the end of string */ - DISPLAYLEVEL(4, "windowLog=%d\nchainLog=%d\nhashLog=%d\nsearchLog=%d\n", params->windowLog, params->chainLog, params->hashLog, params->searchLog); - DISPLAYLEVEL(4, "searchLength=%d\ntargetLength=%d\nstrategy=%d\n", params->searchLength, params->targetLength, params->strategy); return 1; } @@ -364,27 +367,28 @@ typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom int main(int argCount, const char* argv[]) { int argNb, - forceStdout=0, - followLinks=0, - main_pause=0, - nextEntryIsDictionary=0, - operationResult=0, - nextArgumentIsOutFileName=0, - nextArgumentIsMaxDict=0, - nextArgumentIsDictID=0, - nextArgumentsAreFiles=0, - ultra=0, + followLinks = 0, + forceStdout = 0, lastCommand = 0, - nbThreads = 1, - setRealTimePrio = 0, + ldmFlag = 0, + main_pause = 0, + nbWorkers = 0, + nextArgumentIsOutFileName = 0, + nextArgumentIsMaxDict = 0, + nextArgumentIsDictID = 0, + nextArgumentsAreFiles = 0, + nextEntryIsDictionary = 0, + operationResult = 0, separateFiles = 0, - ldmFlag = 0; + setRealTimePrio = 0, + singleThread = 0, + ultra=0; unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */ size_t blockSize = 0; zstd_operation_mode operation = zom_compress; ZSTD_compressionParameters compressionParams; int cLevel = ZSTDCLI_CLEVEL_DEFAULT; - int cLevelLast = 1; + int cLevelLast = -1000000000; unsigned recursive = 0; unsigned memLimit = 0; const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ @@ -416,22 +420,25 @@ int main(int argCount, const char* argv[]) if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); } filenameTable[0] = stdinmark; g_displayOut = stderr; - programName = lastNameFromPath(programName); +#ifdef ZSTD_MULTITHREAD + nbWorkers = 1; +#endif /* preset behaviors */ - if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbThreads=0; + if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0; if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress; - if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } - if (exeNameMatch(programName, ZSTD_GZ)) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); FIO_setRemoveSrcFile(1); } /* behave like gzip */ - if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(1); } /* behave like gunzip */ - if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat */ - if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); } /* behave like lzma */ - if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); } /* behave like unlzma */ - if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like xz */ - if (exeNameMatch(programName, ZSTD_UNXZ)) { operation=zom_decompress; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like unxz */ - if (exeNameMatch(programName, ZSTD_LZ4)) { suffix = LZ4_EXTENSION; FIO_setCompressionType(FIO_lz4Compression); FIO_setRemoveSrcFile(1); } /* behave like xz */ - if (exeNameMatch(programName, ZSTD_UNLZ4)) { operation=zom_decompress; FIO_setCompressionType(FIO_lz4Compression); FIO_setRemoveSrcFile(1); } /* behave like unxz */ + if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* supports multiple formats */ + if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* behave like zcat, also supports multiple formats */ + if (exeNameMatch(programName, ZSTD_GZ)) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); FIO_setRemoveSrcFile(1); } /* behave like gzip */ + if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(1); } /* behave like gunzip, also supports multiple formats */ + if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat, also supports multiple formats */ + if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); } /* behave like lzma */ + if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); } /* behave like unlzma, also supports multiple formats */ + if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like xz */ + if (exeNameMatch(programName, ZSTD_UNXZ)) { operation=zom_decompress; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like unxz, also supports multiple formats */ + if (exeNameMatch(programName, ZSTD_LZ4)) { suffix = LZ4_EXTENSION; FIO_setCompressionType(FIO_lz4Compression); } /* behave like lz4 */ + if (exeNameMatch(programName, ZSTD_UNLZ4)) { operation=zom_decompress; FIO_setCompressionType(FIO_lz4Compression); } /* behave like unlz4, also supports multiple formats */ memset(&compressionParams, 0, sizeof(compressionParams)); /* command switches */ @@ -478,6 +485,7 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; } + if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; } #ifdef ZSTD_GZCOMPRESS if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); continue; } #endif @@ -512,7 +520,7 @@ int main(int argCount, const char* argv[]) continue; } #endif - if (longCommandWArg(&argument, "--threads=")) { nbThreads = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "--threads=")) { nbWorkers = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; } @@ -536,6 +544,21 @@ int main(int argCount, const char* argv[]) compressionParams.windowLog = ldmWindowLog; continue; } + if (longCommandWArg(&argument, "--fast")) { + /* Parse optional window log */ + if (*argument == '=') { + U32 fastLevel; + ++argument; + fastLevel = readU32FromChar(&argument); + if (fastLevel) cLevel = - (int)fastLevel; + } else if (*argument != 0) { + /* Invalid character following --fast */ + CLEAN_RETURN(badusage(programName)); + } else { + cLevel = -1; /* default for --fast */ + } + continue; + } /* fall-through, will trigger bad_usage() later on */ } @@ -566,7 +589,8 @@ int main(int argCount, const char* argv[]) /* Decoding */ case 'd': #ifndef ZSTD_NOBENCH - if (operation==zom_bench) { BMK_setDecodeOnlyMode(1); argument++; break; } /* benchmark decode (hidden option) */ + BMK_setDecodeOnlyMode(1); + if (operation==zom_bench) { argument++; break; } /* benchmark decode (hidden option) */ #endif operation=zom_decompress; argument++; break; @@ -645,7 +669,7 @@ int main(int argCount, const char* argv[]) /* nb of threads (hidden option) */ case 'T': argument++; - nbThreads = readU32FromChar(&argument); + nbWorkers = readU32FromChar(&argument); break; /* Dictionary Selection level */ @@ -713,11 +737,13 @@ int main(int argCount, const char* argv[]) /* Welcome message (if verbose) */ DISPLAYLEVEL(3, WELCOME_MESSAGE); - if (nbThreads == 0) { - /* try to guess */ - nbThreads = UTIL_countPhysicalCores(); - DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbThreads); +#ifdef ZSTD_MULTITHREAD + if ((nbWorkers==0) && (!singleThread)) { + /* automatically set # workers based on # of reported cpus */ + nbWorkers = UTIL_countPhysicalCores(); + DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers); } +#endif g_utilDisplayLevel = g_displayLevel; if (!followLinks) { @@ -760,7 +786,7 @@ int main(int argCount, const char* argv[]) BMK_setNotificationLevel(g_displayLevel); BMK_setSeparateFiles(separateFiles); BMK_setBlockSize(blockSize); - BMK_setNbThreads(nbThreads); + BMK_setNbWorkers(nbWorkers); BMK_setRealTime(setRealTimePrio); BMK_setNbSeconds(bench_nbSeconds); BMK_setLdmFlag(ldmFlag); @@ -788,7 +814,7 @@ int main(int argCount, const char* argv[]) zParams.dictID = dictID; if (cover) { int const optimize = !coverParams.k || !coverParams.d; - coverParams.nbThreads = nbThreads; + coverParams.nbThreads = nbWorkers; coverParams.zParams = zParams; operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, optimize); } else { @@ -803,16 +829,22 @@ int main(int argCount, const char* argv[]) } #ifndef ZSTD_NODECOMPRESS - if (operation==zom_test) { outFileName=nulmark; FIO_setRemoveSrcFile(0); } /* test mode */ + if (operation==zom_test) { outFileName=nulmark; FIO_setRemoveSrcFile(0); } /* test mode */ #endif /* No input filename ==> use stdin and stdout */ filenameIdx += !filenameIdx; /* filenameTable[0] is stdin by default */ - if (!strcmp(filenameTable[0], stdinmark) && !outFileName) outFileName = stdoutmark; /* when input is stdin, default output is stdout */ + if (!strcmp(filenameTable[0], stdinmark) && !outFileName) + outFileName = stdoutmark; /* when input is stdin, default output is stdout */ /* Check if input/output defined as console; trigger an error in this case */ - if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) ) CLEAN_RETURN(badusage(programName)); - if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && !strcmp(filenameTable[0], stdinmark) && !forceStdout && operation!=zom_decompress) + if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) ) + CLEAN_RETURN(badusage(programName)); + if ( outFileName && !strcmp(outFileName, stdoutmark) + && IS_CONSOLE(stdout) + && !strcmp(filenameTable[0], stdinmark) + && !forceStdout + && operation!=zom_decompress ) CLEAN_RETURN(badusage(programName)); #ifndef ZSTD_NOCOMPRESS @@ -832,7 +864,7 @@ int main(int argCount, const char* argv[]) FIO_setNotificationLevel(g_displayLevel); if (operation==zom_compress) { #ifndef ZSTD_NOCOMPRESS - FIO_setNbThreads(nbThreads); + FIO_setNbWorkers(nbWorkers); FIO_setBlockSize((U32)blockSize); FIO_setLdmFlag(ldmFlag); FIO_setLdmHashLog(g_ldmHashLog); |