diff options
Diffstat (limited to 'programs/zstdcli.c')
| -rw-r--r-- | programs/zstdcli.c | 120 |
1 files changed, 67 insertions, 53 deletions
diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 64dad110c7af5..fbb1c048a363b 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -148,6 +148,7 @@ static int usage_advanced(const char* programName) #endif DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n"); DISPLAY( "--[no-]check : integrity check (default: enabled) \n"); + DISPLAY( "--[no-]compress-literals : force (un)compressed literals \n"); #endif #ifdef UTIL_HAS_CREATEFILELIST DISPLAY( " -r : operate recursively on directories \n"); @@ -241,18 +242,20 @@ static void errorOut(const char* msg) * @return 1 if an overflow error occurs */ static int readU32FromCharChecked(const char** stringPtr, unsigned* value) { - static unsigned const max = (((unsigned)(-1)) / 10) - 1; unsigned result = 0; while ((**stringPtr >='0') && (**stringPtr <='9')) { - if (result > max) return 1; // overflow error - result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + unsigned const max = (((unsigned)(-1)) / 10) - 1; + if (result > max) return 1; /* overflow error */ + result *= 10; + result += (unsigned)(**stringPtr - '0'); + (*stringPtr)++ ; } if ((**stringPtr=='K') || (**stringPtr=='M')) { unsigned const maxK = ((unsigned)(-1)) >> 10; - if (result > maxK) return 1; // overflow error + if (result > maxK) return 1; /* overflow error */ result <<= 10; if (**stringPtr=='M') { - if (result > maxK) return 1; // overflow error + if (result > maxK) return 1; /* overflow error */ result <<= 10; } (*stringPtr)++; /* skip `K` or `M` */ @@ -344,7 +347,7 @@ static unsigned parseFastCoverParameters(const char* stringPtr, ZDICT_fastCover_ /** * parseLegacyParameters() : - * reads legacy dictioanry builter parameters from *stringPtr (e.g. "--train-legacy=selectivity=8") into *selectivity + * reads legacy dictionary builder parameters from *stringPtr (e.g. "--train-legacy=selectivity=8") into *selectivity * @return 1 means that legacy dictionary builder parameters were correct * @return 0 in case of malformed parameters */ @@ -483,7 +486,7 @@ static int init_cLevel(void) { if ((*ptr>='0') && (*ptr<='9')) { unsigned absLevel; - if (readU32FromCharChecked(&ptr, &absLevel)) { + if (readU32FromCharChecked(&ptr, &absLevel)) { DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large\n", ENV_CLEVEL, env); return ZSTDCLI_CLEVEL_DEFAULT; } else if (*ptr == 0) { @@ -536,6 +539,8 @@ int main(int argCount, const char* argv[]) double compressibility = 0.5; unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */ size_t blockSize = 0; + + FIO_prefs_t* const prefs = FIO_createPreferences(); zstd_operation_mode operation = zom_compress; ZSTD_compressionParameters compressionParams; int cLevel; @@ -565,6 +570,7 @@ int main(int argCount, const char* argv[]) #ifndef ZSTD_NOBENCH BMK_advancedParams_t benchParams = BMK_initAdvancedParams(); #endif + ZSTD_literalCompressionMode_e literalCompressionMode = ZSTD_lcm_auto; /* init */ @@ -582,17 +588,17 @@ int main(int argCount, const char* argv[]) /* preset behaviors */ if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0; if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress; - if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* supports multiple formats */ - if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* behave like zcat, also supports multiple formats */ - if (exeNameMatch(programName, ZSTD_GZ)) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); FIO_setRemoveSrcFile(1); } /* behave like gzip */ - if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(1); } /* behave like gunzip, also supports multiple formats */ - if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat, also supports multiple formats */ - if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); } /* behave like lzma */ - if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); } /* behave like unlzma, also supports multiple formats */ - if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like xz */ - if (exeNameMatch(programName, ZSTD_UNXZ)) { operation=zom_decompress; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like unxz, also supports multiple formats */ - if (exeNameMatch(programName, ZSTD_LZ4)) { suffix = LZ4_EXTENSION; FIO_setCompressionType(FIO_lz4Compression); } /* behave like lz4 */ - if (exeNameMatch(programName, ZSTD_UNLZ4)) { operation=zom_decompress; FIO_setCompressionType(FIO_lz4Compression); } /* behave like unlz4, also supports multiple formats */ + if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(prefs); outFileName=stdoutmark; g_displayLevel=1; } /* supports multiple formats */ + if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(prefs); outFileName=stdoutmark; g_displayLevel=1; } /* behave like zcat, also supports multiple formats */ + if (exeNameMatch(programName, ZSTD_GZ)) { suffix = GZ_EXTENSION; FIO_setCompressionType(prefs, FIO_gzipCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like gzip */ + if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(prefs, 1); } /* behave like gunzip, also supports multiple formats */ + if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(prefs); outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat, also supports multiple formats */ + if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(prefs, FIO_lzmaCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like lzma */ + if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_lzmaCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like unlzma, also supports multiple formats */ + if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(prefs, FIO_xzCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like xz */ + if (exeNameMatch(programName, ZSTD_UNXZ)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_xzCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like unxz, also supports multiple formats */ + if (exeNameMatch(programName, ZSTD_LZ4)) { suffix = LZ4_EXTENSION; FIO_setCompressionType(prefs, FIO_lz4Compression); } /* behave like lz4 */ + if (exeNameMatch(programName, ZSTD_UNLZ4)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_lz4Compression); } /* behave like unlz4, also supports multiple formats */ memset(&compressionParams, 0, sizeof(compressionParams)); /* init crash handler */ @@ -623,40 +629,42 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; } if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; } if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; } - if (!strcmp(argument, "--force")) { FIO_overwriteMode(); forceStdout=1; followLinks=1; continue; } + if (!strcmp(argument, "--force")) { FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; continue; } if (!strcmp(argument, "--version")) { g_displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); } if (!strcmp(argument, "--help")) { g_displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); } if (!strcmp(argument, "--verbose")) { g_displayLevel++; continue; } if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; } if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; g_displayLevel-=(g_displayLevel==2); continue; } if (!strcmp(argument, "--ultra")) { ultra=1; continue; } - if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; } - if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; } - if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; } - if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; } + if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(prefs, 2); continue; } + if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(prefs, 0); continue; } + if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(prefs, 2); continue; } + if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(prefs, 0); continue; } if (!strcmp(argument, "--test")) { operation=zom_test; continue; } if (!strcmp(argument, "--train")) { operation=zom_train; if (outFileName==NULL) outFileName=g_defaultDictName; continue; } if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; lastCommand=1; continue; } /* kept available for compatibility with old syntax ; will be removed one day */ if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; lastCommand=1; continue; } /* kept available for compatibility with old syntax ; will be removed one day */ - if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; } - if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; } - if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } + if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(prefs, 0); continue; } + if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(prefs, 0); continue; } + if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(prefs, 1); continue; } if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; } if (!strcmp(argument, "--adapt")) { adapt = 1; continue; } if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) CLEAN_RETURN(badusage(programName)); continue; } if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; } - if (!strcmp(argument, "--format=zstd")) { suffix = ZSTD_EXTENSION; FIO_setCompressionType(FIO_zstdCompression); continue; } + if (!strcmp(argument, "--format=zstd")) { suffix = ZSTD_EXTENSION; FIO_setCompressionType(prefs, FIO_zstdCompression); continue; } #ifdef ZSTD_GZCOMPRESS - if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); continue; } + if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(prefs, FIO_gzipCompression); continue; } #endif #ifdef ZSTD_LZMACOMPRESS - if (!strcmp(argument, "--format=lzma")) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression); continue; } - if (!strcmp(argument, "--format=xz")) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); continue; } + if (!strcmp(argument, "--format=lzma")) { suffix = LZMA_EXTENSION; FIO_setCompressionType(prefs, FIO_lzmaCompression); continue; } + if (!strcmp(argument, "--format=xz")) { suffix = XZ_EXTENSION; FIO_setCompressionType(prefs, FIO_xzCompression); continue; } #endif #ifdef ZSTD_LZ4COMPRESS - if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; FIO_setCompressionType(FIO_lz4Compression); continue; } + if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; FIO_setCompressionType(prefs, FIO_lz4Compression); continue; } #endif if (!strcmp(argument, "--rsyncable")) { rsyncable = 1; continue; } + if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_lcm_huffman; continue; } + if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_lcm_uncompressed; continue; } if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; } /* long commands with arguments */ @@ -784,7 +792,7 @@ int main(int argCount, const char* argv[]) case 'D': nextEntryIsDictionary = 1; lastCommand = 1; argument++; break; /* Overwrite */ - case 'f': FIO_overwriteMode(); forceStdout=1; followLinks=1; argument++; break; + case 'f': FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; argument++; break; /* Verbose mode */ case 'v': g_displayLevel++; argument++; break; @@ -793,10 +801,10 @@ int main(int argCount, const char* argv[]) case 'q': g_displayLevel--; argument++; break; /* keep source file (default) */ - case 'k': FIO_setRemoveSrcFile(0); argument++; break; + case 'k': FIO_setRemoveSrcFile(prefs, 0); argument++; break; /* Checksum */ - case 'C': FIO_setChecksumFlag(2); argument++; break; + case 'C': FIO_setChecksumFlag(prefs, 2); argument++; break; /* test compressed file */ case 't': operation=zom_test; argument++; break; @@ -949,6 +957,8 @@ int main(int argCount, const char* argv[]) filenameTable[fileNamesNb++] = filenameTable[u]; } } + if (fileNamesNb == 0 && filenameIdx > 0) + CLEAN_RETURN(1); filenameIdx = fileNamesNb; } if (recursive) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */ @@ -991,6 +1001,7 @@ int main(int argCount, const char* argv[]) if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) { benchParams.ldmHashRateLog = g_ldmHashRateLog; } + benchParams.literalCompressionMode = literalCompressionMode; if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel(); if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel(); @@ -1057,7 +1068,7 @@ int main(int argCount, const char* argv[]) } #ifndef ZSTD_NODECOMPRESS - if (operation==zom_test) { outFileName=nulmark; FIO_setRemoveSrcFile(0); } /* test mode */ + if (operation==zom_test) { outFileName=nulmark; FIO_setRemoveSrcFile(prefs, 0); } /* test mode */ #endif /* No input filename ==> use stdin and stdout */ @@ -1092,27 +1103,28 @@ int main(int argCount, const char* argv[]) FIO_setNotificationLevel(g_displayLevel); if (operation==zom_compress) { #ifndef ZSTD_NOCOMPRESS - FIO_setNbWorkers(nbWorkers); - FIO_setBlockSize((U32)blockSize); - if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(g_overlapLog); - FIO_setLdmFlag(ldmFlag); - FIO_setLdmHashLog(g_ldmHashLog); - FIO_setLdmMinMatch(g_ldmMinMatch); - if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(g_ldmBucketSizeLog); - if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(g_ldmHashRateLog); - FIO_setAdaptiveMode(adapt); - FIO_setAdaptMin(adaptMin); - FIO_setAdaptMax(adaptMax); - FIO_setRsyncable(rsyncable); + FIO_setNbWorkers(prefs, nbWorkers); + FIO_setBlockSize(prefs, (U32)blockSize); + if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, g_overlapLog); + FIO_setLdmFlag(prefs, ldmFlag); + FIO_setLdmHashLog(prefs, g_ldmHashLog); + FIO_setLdmMinMatch(prefs, g_ldmMinMatch); + if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, g_ldmBucketSizeLog); + if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, g_ldmHashRateLog); + FIO_setAdaptiveMode(prefs, adapt); + FIO_setAdaptMin(prefs, adaptMin); + FIO_setAdaptMax(prefs, adaptMax); + FIO_setRsyncable(prefs, rsyncable); + FIO_setLiteralCompressionMode(prefs, literalCompressionMode); if (adaptMin > cLevel) cLevel = adaptMin; if (adaptMax < cLevel) cLevel = adaptMax; if ((filenameIdx==1) && outFileName) - operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, compressionParams); + operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams); else - operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams); + operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams); #else - (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; /* not used when ZSTD_NOCOMPRESS set */ + (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; /* not used when ZSTD_NOCOMPRESS set */ DISPLAY("Compression not supported \n"); #endif } else { /* decompression or test */ @@ -1124,17 +1136,19 @@ int main(int argCount, const char* argv[]) memLimit = (U32)1 << (compressionParams.windowLog & 31); } } - FIO_setMemLimit(memLimit); + FIO_setMemLimit(prefs, memLimit); if (filenameIdx==1 && outFileName) - operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName); + operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName); else - operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, outFileName, dictFileName); + operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outFileName, dictFileName); #else DISPLAY("Decompression not supported \n"); #endif } _end: + FIO_freePreferences(prefs); + if (main_pause) waitEnter(); #ifdef UTIL_HAS_CREATEFILELIST if (extendedFileList) |