diff options
Diffstat (limited to 'programs/zstdcli.c')
| -rw-r--r-- | programs/zstdcli.c | 91 |
1 files changed, 58 insertions, 33 deletions
diff --git a/programs/zstdcli.c b/programs/zstdcli.c index de286cdf283e..cd6b40bc089f 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -136,16 +136,19 @@ static int usage_advanced(const char* programName) DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); DISPLAY( " -c : force write to standard output, even if it is the console\n"); DISPLAY( " -l : print information about zstd compressed files \n"); + DISPLAY( "--exclude-compressed: only compress files that are not previously compressed \n"); #ifndef ZSTD_NOCOMPRESS DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel()); DISPLAY( "--long[=#]: enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog); - DISPLAY( "--fast[=#]: switch to ultra fast compression level (default: %u)\n", 1); + DISPLAY( "--fast[=#]: switch to very fast compression levels (default: %u)\n", 1); DISPLAY( "--adapt : dynamically adapt compression level to I/O conditions \n"); + DISPLAY( "--stream-size=# : optimize compression parameters for streaming input of given number of bytes \n"); + DISPLAY( "--size-hint=# optimize compression parameters for streaming input of approximately this size\n"); DISPLAY( "--target-compressed-block-size=# : make compressed block near targeted size \n"); #ifdef ZSTD_MULTITHREAD DISPLAY( " -T# : spawns # compression threads (default: 1, 0==# cores) \n"); DISPLAY( " -B# : select size of each job (default: 0==automatic) \n"); - DISPLAY( " --rsyncable : compress using a rsync-friendly method (-B sets block size) \n"); + DISPLAY( "--rsyncable : compress using a rsync-friendly method (-B sets block size) \n"); #endif DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n"); DISPLAY( "--[no-]check : integrity check (default: enabled) \n"); @@ -153,6 +156,7 @@ static int usage_advanced(const char* programName) #endif #ifdef UTIL_HAS_CREATEFILELIST DISPLAY( " -r : operate recursively on directories \n"); + DISPLAY( "--output-dir-flat[=directory]: all resulting files stored into `directory`. \n"); #endif DISPLAY( "--format=zstd : compress files to the .zst format (default) \n"); #ifdef ZSTD_GZCOMPRESS @@ -284,7 +288,7 @@ static unsigned readU32FromChar(const char** stringPtr) { * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. * @return 0 and doesn't modify *stringPtr otherwise. */ -static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) +static int longCommandWArg(const char** stringPtr, const char* longCommand) { size_t const comSize = strlen(longCommand); int const result = !strncmp(*stringPtr, longCommand, comSize); @@ -427,8 +431,8 @@ static ZDICT_fastCover_params_t defaultFastCoverParams(void) static unsigned parseAdaptParameters(const char* stringPtr, int* adaptMinPtr, int* adaptMaxPtr) { for ( ; ;) { - if (longCommandWArg(&stringPtr, "min=")) { *adaptMinPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } - if (longCommandWArg(&stringPtr, "max=")) { *adaptMaxPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "min=")) { *adaptMinPtr = (int)readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "max=")) { *adaptMaxPtr = (int)readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } DISPLAYLEVEL(4, "invalid compression parameter \n"); return 0; } @@ -523,7 +527,7 @@ static int init_cLevel(void) { DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large\n", ENV_CLEVEL, env); return ZSTDCLI_CLEVEL_DEFAULT; } else if (*ptr == 0) { - return sign * absLevel; + return sign * (int)absLevel; } } @@ -560,6 +564,7 @@ int main(int argCount, const char* argv[]) adaptMax = MAXCLEVEL, rsyncable = 0, nextArgumentIsOutFileName = 0, + nextArgumentIsOutDirName = 0, nextArgumentIsMaxDict = 0, nextArgumentIsDictID = 0, nextArgumentsAreFiles = 0, @@ -580,15 +585,18 @@ int main(int argCount, const char* argv[]) int cLevelLast = -1000000000; unsigned recursive = 0; unsigned memLimit = 0; - const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ + const char** filenameTable = (const char**)malloc((size_t)argCount * sizeof(const char*)); /* argCount >= 1 */ unsigned filenameIdx = 0; const char* programName = argv[0]; const char* outFileName = NULL; + const char* outDirName = NULL; const char* dictFileName = NULL; const char* suffix = ZSTD_EXTENSION; unsigned maxDictSize = g_defaultMaxDictSize; unsigned dictID = 0; + size_t streamSrcSize = 0; size_t targetCBlockSize = 0; + size_t srcSizeHint = 0; int dictCLevel = g_defaultDictCLevel; unsigned dictSelect = g_defaultSelectivityLevel; #ifdef UTIL_HAS_CREATEFILELIST @@ -682,6 +690,7 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(prefs, 0); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(prefs, 1); continue; } if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; } + if (!strcmp(argument, "--output-dir-flat")) {nextArgumentIsOutDirName=1; lastCommand=1; continue; } if (!strcmp(argument, "--adapt")) { adapt = 1; continue; } if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) CLEAN_RETURN(badusage(programName)); continue; } if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; } @@ -700,7 +709,7 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_lcm_huffman; continue; } if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_lcm_uncompressed; continue; } if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; } - + if (!strcmp(argument, "--exclude-compressed")) { FIO_setExcludeCompressedFile(prefs, 1); continue; } /* long commands with arguments */ #ifndef ZSTD_NODICT if (longCommandWArg(&argument, "--train-cover")) { @@ -737,7 +746,7 @@ int main(int argCount, const char* argv[]) continue; } #endif - if (longCommandWArg(&argument, "--threads=")) { nbWorkers = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "--threads=")) { nbWorkers = (int)readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; } @@ -745,7 +754,10 @@ int main(int argCount, const char* argv[]) if (longCommandWArg(&argument, "--maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--dictID=")) { dictID = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) CLEAN_RETURN(badusage(programName)); continue; } + if (longCommandWArg(&argument, "--stream-size=")) { streamSrcSize = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--target-compressed-block-size=")) { targetCBlockSize = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "--size-hint=")) { srcSizeHint = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "--output-dir-flat=")) { outDirName = argument; continue; } if (longCommandWArg(&argument, "--long")) { unsigned ldmWindowLog = 0; ldmFlag = 1; @@ -797,7 +809,7 @@ int main(int argCount, const char* argv[]) #ifndef ZSTD_NOCOMPRESS /* compression Level */ if ((*argument>='0') && (*argument<='9')) { - dictCLevel = cLevel = readU32FromChar(&argument); + dictCLevel = cLevel = (int)readU32FromChar(&argument); continue; } #endif @@ -869,7 +881,7 @@ int main(int argCount, const char* argv[]) case 'e': /* compression Level */ argument++; - cLevelLast = readU32FromChar(&argument); + cLevelLast = (int)readU32FromChar(&argument); break; /* Modify Nb Iterations (benchmark only) */ @@ -895,7 +907,7 @@ int main(int argCount, const char* argv[]) /* nb of threads (hidden option) */ case 'T': argument++; - nbWorkers = readU32FromChar(&argument); + nbWorkers = (int)readU32FromChar(&argument); break; /* Dictionary Selection level */ @@ -959,6 +971,13 @@ int main(int argCount, const char* argv[]) continue; } + if (nextArgumentIsOutDirName) { + nextArgumentIsOutDirName = 0; + lastCommand = 0; + outDirName = argument; + continue; + } + /* add filename to list */ filenameTable[filenameIdx++] = argument; } @@ -986,7 +1005,11 @@ int main(int argCount, const char* argv[]) if (!followLinks) { unsigned u; for (u=0, fileNamesNb=0; u<filenameIdx; u++) { - if (UTIL_isLink(filenameTable[u])) { + if (UTIL_isLink(filenameTable[u]) +#ifndef _MSC_VER + && !UTIL_isFIFO(filenameTable[u]) +#endif /* _MSC_VER */ + ) { DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", filenameTable[u]); } else { filenameTable[fileNamesNb++] = filenameTable[u]; @@ -1025,16 +1048,16 @@ int main(int argCount, const char* argv[]) #ifndef ZSTD_NOBENCH benchParams.blockSize = blockSize; benchParams.nbWorkers = nbWorkers; - benchParams.realTime = setRealTimePrio; + benchParams.realTime = (unsigned)setRealTimePrio; benchParams.nbSeconds = bench_nbSeconds; benchParams.ldmFlag = ldmFlag; - benchParams.ldmMinMatch = g_ldmMinMatch; - benchParams.ldmHashLog = g_ldmHashLog; + benchParams.ldmMinMatch = (int)g_ldmMinMatch; + benchParams.ldmHashLog = (int)g_ldmHashLog; if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) { - benchParams.ldmBucketSizeLog = g_ldmBucketSizeLog; + benchParams.ldmBucketSizeLog = (int)g_ldmBucketSizeLog; } if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) { - benchParams.ldmHashRateLog = g_ldmHashRateLog; + benchParams.ldmHashRateLog = (int)g_ldmHashRateLog; } benchParams.literalCompressionMode = literalCompressionMode; @@ -1075,16 +1098,16 @@ int main(int argCount, const char* argv[]) #ifndef ZSTD_NODICT ZDICT_params_t zParams; zParams.compressionLevel = dictCLevel; - zParams.notificationLevel = g_displayLevel; + zParams.notificationLevel = (unsigned)g_displayLevel; zParams.dictID = dictID; if (dict == cover) { int const optimize = !coverParams.k || !coverParams.d; - coverParams.nbThreads = nbWorkers; + coverParams.nbThreads = (unsigned)nbWorkers; coverParams.zParams = zParams; operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, NULL, optimize); } else if (dict == fastCover) { int const optimize = !fastCoverParams.k || !fastCoverParams.d; - fastCoverParams.nbThreads = nbWorkers; + fastCoverParams.nbThreads = (unsigned)nbWorkers; fastCoverParams.zParams = zParams; operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, NULL, &fastCoverParams, optimize); } else { @@ -1103,7 +1126,7 @@ int main(int argCount, const char* argv[]) } #ifndef ZSTD_NODECOMPRESS - if (operation==zom_test) { outFileName=nulmark; FIO_setRemoveSrcFile(prefs, 0); } /* test mode */ + if (operation==zom_test) { FIO_setTestMode(prefs, 1); outFileName=nulmark; FIO_setRemoveSrcFile(prefs, 0); } /* test mode */ #endif /* No input filename ==> use stdin and stdout */ @@ -1139,18 +1162,20 @@ int main(int argCount, const char* argv[]) if (operation==zom_compress) { #ifndef ZSTD_NOCOMPRESS FIO_setNbWorkers(prefs, nbWorkers); - FIO_setBlockSize(prefs, (U32)blockSize); - if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, g_overlapLog); - FIO_setLdmFlag(prefs, ldmFlag); - FIO_setLdmHashLog(prefs, g_ldmHashLog); - FIO_setLdmMinMatch(prefs, g_ldmMinMatch); - if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, g_ldmBucketSizeLog); - if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, g_ldmHashRateLog); - FIO_setAdaptiveMode(prefs, adapt); + FIO_setBlockSize(prefs, (int)blockSize); + if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, (int)g_overlapLog); + FIO_setLdmFlag(prefs, (unsigned)ldmFlag); + FIO_setLdmHashLog(prefs, (int)g_ldmHashLog); + FIO_setLdmMinMatch(prefs, (int)g_ldmMinMatch); + if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, (int)g_ldmBucketSizeLog); + if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, (int)g_ldmHashRateLog); + FIO_setAdaptiveMode(prefs, (unsigned)adapt); FIO_setAdaptMin(prefs, adaptMin); FIO_setAdaptMax(prefs, adaptMax); FIO_setRsyncable(prefs, rsyncable); + FIO_setStreamSrcSize(prefs, streamSrcSize); FIO_setTargetCBlockSize(prefs, targetCBlockSize); + FIO_setSrcSizeHint(prefs, srcSizeHint); FIO_setLiteralCompressionMode(prefs, literalCompressionMode); if (adaptMin > cLevel) cLevel = adaptMin; if (adaptMax < cLevel) cLevel = adaptMax; @@ -1158,9 +1183,9 @@ int main(int argCount, const char* argv[]) if ((filenameIdx==1) && outFileName) operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams); else - operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams); + operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams); #else - (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; /* not used when ZSTD_NOCOMPRESS set */ + (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; /* not used when ZSTD_NOCOMPRESS set */ DISPLAY("Compression not supported \n"); #endif } else { /* decompression or test */ @@ -1176,7 +1201,7 @@ int main(int argCount, const char* argv[]) if (filenameIdx==1 && outFileName) operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName); else - operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outFileName, dictFileName); + operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, outFileName, dictFileName); #else DISPLAY("Decompression not supported \n"); #endif |