diff options
Diffstat (limited to 'programs/zstdcli.c')
| -rw-r--r-- | programs/zstdcli.c | 195 |
1 files changed, 111 insertions, 84 deletions
diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 70e2b70666bf..9b6f91533462 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -20,7 +20,9 @@ # define ZSTDCLI_CLEVEL_MAX 19 /* without using --ultra */ #endif - +#ifndef ZSTDCLI_NBTHREADS_DEFAULT +# define ZSTDCLI_NBTHREADS_DEFAULT 1 +#endif /*-************************************ * Dependencies @@ -146,9 +148,26 @@ static void usage_advanced(const char* programName) #ifdef UTIL_HAS_CREATEFILELIST DISPLAYOUT( " -r : operate recursively on directories \n"); - DISPLAYOUT( "--filelist=FILE : read list of files to operate upon from FILE \n"); - DISPLAYOUT( "--output-dir-flat=DIR : all resulting files are stored into DIR \n"); + DISPLAYOUT( "--filelist FILE : read list of files to operate upon from FILE \n"); + DISPLAYOUT( "--output-dir-flat DIR : processed files are stored into DIR \n"); +#endif + +#ifdef UTIL_HAS_MIRRORFILELIST + DISPLAYOUT( "--output-dir-mirror DIR : processed files are stored into DIR respecting original directory structure \n"); +#endif + + +#ifndef ZSTD_NOCOMPRESS + DISPLAYOUT( "--[no-]check : during compression, add XXH64 integrity checksum to frame (default: enabled)"); +#ifndef ZSTD_NODECOMPRESS + DISPLAYOUT( ". If specified with -d, decompressor will ignore/validate checksums in compressed frame (default: validate)."); #endif +#else +#ifdef ZSTD_NOCOMPRESS + DISPLAYOUT( "--[no-]check : during decompression, ignore/validate checksums in compressed frame (default: validate)."); +#endif +#endif /* ZSTD_NOCOMPRESS */ + DISPLAYOUT( "\n"); DISPLAYOUT( "-- : All arguments after \"--\" are treated as files \n"); @@ -170,7 +189,6 @@ static void usage_advanced(const char* programName) DISPLAYOUT( "--size-hint=# optimize compression parameters for streaming input of approximately this size \n"); DISPLAYOUT( "--target-compressed-block-size=# : generate compressed block of approximately targeted size \n"); DISPLAYOUT( "--no-dictID : don't write dictID into header (dictionary compression only) \n"); - DISPLAYOUT( "--[no-]check : add XXH64 integrity checksum to frame (default: enabled) \n"); DISPLAYOUT( "--[no-]compress-literals : force (un)compressed literals \n"); DISPLAYOUT( "--format=zstd : compress files to the .zst format (default) \n"); @@ -544,6 +562,11 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi static void printVersion(void) { + if (g_displayLevel < DISPLAY_LEVEL_DEFAULT) { + DISPLAYOUT("%s\n", ZSTD_VERSION_STRING); + return; + } + DISPLAYOUT(WELCOME_MESSAGE); if (g_displayLevel >= 3) { /* format support */ @@ -577,6 +600,7 @@ static void printVersion(void) /* Environment variables for parameter setting */ #define ENV_CLEVEL "ZSTD_CLEVEL" +#define ENV_NBTHREADS "ZSTD_NBTHREADS" /* takes lower precedence than directly specifying -T# in the CLI */ /* pick up environment variable */ static int init_cLevel(void) { @@ -606,8 +630,51 @@ static int init_cLevel(void) { return ZSTDCLI_CLEVEL_DEFAULT; } -#define ZSTD_NB_STRATEGIES 9 +#ifdef ZSTD_MULTITHREAD +static unsigned init_nbThreads(void) { + const char* const env = getenv(ENV_NBTHREADS); + if (env != NULL) { + const char* ptr = env; + if ((*ptr>='0') && (*ptr<='9')) { + unsigned nbThreads; + if (readU32FromCharChecked(&ptr, &nbThreads)) { + DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large \n", ENV_NBTHREADS, env); + return ZSTDCLI_NBTHREADS_DEFAULT; + } else if (*ptr == 0) { + return nbThreads; + } + } + DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: not a valid unsigned value \n", ENV_NBTHREADS, env); + } + return ZSTDCLI_NBTHREADS_DEFAULT; +} +#endif + +#define NEXT_FIELD(ptr) { \ + if (*argument == '=') { \ + ptr = ++argument; \ + argument += strlen(ptr); \ + } else { \ + argNb++; \ + if (argNb >= argCount) { \ + DISPLAY("error: missing command argument \n"); \ + CLEAN_RETURN(1); \ + } \ + ptr = argv[argNb]; \ + assert(ptr != NULL); \ + if (ptr[0]=='-') { \ + DISPLAY("error: command cannot be separated from its argument by another command \n"); \ + CLEAN_RETURN(1); \ +} } } + +#define NEXT_UINT32(val32) { \ + const char* __nb; \ + NEXT_FIELD(__nb); \ + val32 = readU32FromChar(&__nb); \ +} + +#define ZSTD_NB_STRATEGIES 9 static const char* ZSTD_strategyMap[ZSTD_NB_STRATEGIES + 1] = { "", "ZSTD_fast", "ZSTD_dfast", "ZSTD_greedy", "ZSTD_lazy", "ZSTD_lazy2", "ZSTD_btlazy2", "ZSTD_btopt", "ZSTD_btultra", "ZSTD_btultra2"}; @@ -630,7 +697,7 @@ int main(int const argCount, const char* argv[]) int argNb, followLinks = 0, forceStdout = 0, - lastCommand = 0, + hasStdout = 0, ldmFlag = 0, main_pause = 0, nbWorkers = 0, @@ -638,12 +705,7 @@ int main(int const argCount, const char* argv[]) adaptMin = MINCLEVEL, adaptMax = MAXCLEVEL, rsyncable = 0, - nextArgumentIsOutFileName = 0, - nextArgumentIsOutDirName = 0, - nextArgumentIsMaxDict = 0, - nextArgumentIsDictID = 0, nextArgumentsAreFiles = 0, - nextEntryIsDictionary = 0, operationResult = 0, separateFiles = 0, setRealTimePrio = 0, @@ -656,6 +718,7 @@ int main(int const argCount, const char* argv[]) size_t blockSize = 0; FIO_prefs_t* const prefs = FIO_createPreferences(); + FIO_ctx_t* const fCtx = FIO_createContext(); zstd_operation_mode operation = zom_compress; ZSTD_compressionParameters compressionParams; int cLevel = init_cLevel(); @@ -667,6 +730,7 @@ int main(int const argCount, const char* argv[]) const char* programName = argv[0]; const char* outFileName = NULL; const char* outDirName = NULL; + const char* outMirroredDirName = NULL; const char* dictFileName = NULL; const char* patchFromDictFileName = NULL; const char* suffix = ZSTD_EXTENSION; @@ -695,7 +759,7 @@ int main(int const argCount, const char* argv[]) if ((filenames==NULL) || (file_of_names==NULL)) { DISPLAY("zstd: allocation error \n"); exit(1); } programName = lastNameFromPath(programName); #ifdef ZSTD_MULTITHREAD - nbWorkers = 1; + nbWorkers = init_nbThreads(); #endif /* preset behaviors */ @@ -756,13 +820,10 @@ int main(int const argCount, const char* argv[]) if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(prefs, 0); continue; } if (!strcmp(argument, "--test")) { operation=zom_test; continue; } if (!strcmp(argument, "--train")) { operation=zom_train; if (outFileName==NULL) outFileName=g_defaultDictName; continue; } - if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; lastCommand=1; continue; } /* kept available for compatibility with old syntax ; will be removed one day */ - if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; lastCommand=1; continue; } /* kept available for compatibility with old syntax ; will be removed one day */ if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(prefs, 0); continue; } if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(prefs, 0); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(prefs, 1); continue; } if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; } - if (!strcmp(argument, "--output-dir-flat")) {nextArgumentIsOutDirName=1; lastCommand=1; continue; } if (!strcmp(argument, "--show-default-cparams")) { showDefaultCParams = 1; continue; } if (!strcmp(argument, "--content-size")) { contentSize = 1; continue; } if (!strcmp(argument, "--no-content-size")) { contentSize = 0; continue; } @@ -785,6 +846,7 @@ int main(int const argCount, const char* argv[]) if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_lcm_uncompressed; continue; } if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; } if (!strcmp(argument, "--exclude-compressed")) { FIO_setExcludeCompressedFile(prefs, 1); continue; } + /* long commands with arguments */ #ifndef ZSTD_NODICT if (longCommandWArg(&argument, "--train-cover")) { @@ -821,19 +883,22 @@ int main(int const argCount, const char* argv[]) continue; } #endif - if (longCommandWArg(&argument, "--threads=")) { nbWorkers = (int)readU32FromChar(&argument); continue; } - if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; } - if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; } - if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "--threads")) { NEXT_UINT32(nbWorkers); continue; } + if (longCommandWArg(&argument, "--memlimit")) { NEXT_UINT32(memLimit); continue; } + if (longCommandWArg(&argument, "--memory")) { NEXT_UINT32(memLimit); continue; } + if (longCommandWArg(&argument, "--memlimit-decompress")) { NEXT_UINT32(memLimit); continue; } if (longCommandWArg(&argument, "--block-size=")) { blockSize = readSizeTFromChar(&argument); continue; } - if (longCommandWArg(&argument, "--maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; } - if (longCommandWArg(&argument, "--dictID=")) { dictID = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "--maxdict")) { NEXT_UINT32(maxDictSize); continue; } + if (longCommandWArg(&argument, "--dictID")) { NEXT_UINT32(dictID); continue; } if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) { badusage(programName); CLEAN_RETURN(1); } continue; } if (longCommandWArg(&argument, "--stream-size=")) { streamSrcSize = readSizeTFromChar(&argument); continue; } if (longCommandWArg(&argument, "--target-compressed-block-size=")) { targetCBlockSize = readSizeTFromChar(&argument); continue; } if (longCommandWArg(&argument, "--size-hint=")) { srcSizeHint = readSizeTFromChar(&argument); continue; } - if (longCommandWArg(&argument, "--output-dir-flat=")) { outDirName = argument; continue; } - if (longCommandWArg(&argument, "--patch-from=")) { patchFromDictFileName = argument; continue; } + if (longCommandWArg(&argument, "--output-dir-flat")) { NEXT_FIELD(outDirName); continue; } +#ifdef UTIL_HAS_MIRRORFILELIST + if (longCommandWArg(&argument, "--output-dir-mirror")) { NEXT_FIELD(outMirroredDirName); continue; } +#endif + if (longCommandWArg(&argument, "--patch-from")) { NEXT_FIELD(patchFromDictFileName); continue; } if (longCommandWArg(&argument, "--long")) { unsigned ldmWindowLog = 0; ldmFlag = 1; @@ -877,8 +942,10 @@ int main(int const argCount, const char* argv[]) } #endif - if (longCommandWArg(&argument, "--filelist=")) { - UTIL_refFilename(file_of_names, argument); + if (longCommandWArg(&argument, "--filelist")) { + const char* listName; + NEXT_FIELD(listName); + UTIL_refFilename(file_of_names, listName); continue; } @@ -887,10 +954,7 @@ int main(int const argCount, const char* argv[]) argument++; while (argument[0]!=0) { - if (lastCommand) { - DISPLAY("error : command must be followed by argument \n"); - CLEAN_RETURN(1); - } + #ifndef ZSTD_NOCOMPRESS /* compression Level */ if ((*argument>='0') && (*argument<='9')) { @@ -921,7 +985,7 @@ int main(int const argCount, const char* argv[]) case 'c': forceStdout=1; outFileName=stdoutmark; argument++; break; /* Use file content as dictionary */ - case 'D': nextEntryIsDictionary = 1; lastCommand = 1; argument++; break; + case 'D': argument++; NEXT_FIELD(dictFileName); break; /* Overwrite */ case 'f': FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; argument++; break; @@ -942,7 +1006,7 @@ int main(int const argCount, const char* argv[]) case 't': operation=zom_test; argument++; break; /* destination file name */ - case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break; + case 'o': argument++; NEXT_FIELD(outFileName); break; /* limit memory */ case 'M': @@ -1013,10 +1077,9 @@ int main(int const argCount, const char* argv[]) /* Select compressibility of synthetic sample */ case 'P': - { argument++; + argument++; compressibility = (double)readU32FromChar(&argument) / 100; - } - break; + break; /* unknown command */ default : badusage(programName); CLEAN_RETURN(1); @@ -1025,51 +1088,10 @@ int main(int const argCount, const char* argv[]) continue; } /* if (argument[0]=='-') */ - if (nextArgumentIsMaxDict) { /* kept available for compatibility with old syntax ; will be removed one day */ - nextArgumentIsMaxDict = 0; - lastCommand = 0; - maxDictSize = readU32FromChar(&argument); - continue; - } - - if (nextArgumentIsDictID) { /* kept available for compatibility with old syntax ; will be removed one day */ - nextArgumentIsDictID = 0; - lastCommand = 0; - dictID = readU32FromChar(&argument); - continue; - } - - if (nextEntryIsDictionary) { - nextEntryIsDictionary = 0; - lastCommand = 0; - dictFileName = argument; - continue; - } - - if (nextArgumentIsOutFileName) { - nextArgumentIsOutFileName = 0; - lastCommand = 0; - outFileName = argument; - if (!strcmp(outFileName, "-")) outFileName = stdoutmark; - continue; - } - - if (nextArgumentIsOutDirName) { - nextArgumentIsOutDirName = 0; - lastCommand = 0; - outDirName = argument; - continue; - } - /* none of the above : add filename to list */ UTIL_refFilename(filenames, argument); } - if (lastCommand) { /* forgotten argument */ - DISPLAY("error : command must be followed by argument \n"); - CLEAN_RETURN(1); - } - /* Welcome message (if verbose) */ DISPLAYLEVEL(3, WELCOME_MESSAGE); @@ -1222,7 +1244,7 @@ int main(int const argCount, const char* argv[]) /* Check if input/output defined as console; trigger an error in this case */ if (!strcmp(filenames->fileNames[0], stdinmark) && IS_CONSOLE(stdin) ) { - badusage(programName); + DISPLAYLEVEL(1, "stdin is a console, aborting\n"); CLEAN_RETURN(1); } if ( outFileName && !strcmp(outFileName, stdoutmark) @@ -1230,7 +1252,7 @@ int main(int const argCount, const char* argv[]) && !strcmp(filenames->fileNames[0], stdinmark) && !forceStdout && operation!=zom_decompress ) { - badusage(programName); + DISPLAYLEVEL(1, "stdout is a console, aborting\n"); CLEAN_RETURN(1); } @@ -1259,12 +1281,16 @@ int main(int const argCount, const char* argv[]) DISPLAY("error : can't use --patch-from=# on multiple files \n"); CLEAN_RETURN(1); } + + /* No status message in pipe mode (stdin - stdout) */ + hasStdout = outFileName && !strcmp(outFileName,stdoutmark); - /* No status message in pipe mode (stdin - stdout) or multi-files mode */ - if (!strcmp(filenames->fileNames[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (g_displayLevel==2)) g_displayLevel=1; - if ((filenames->tableSize > 1) & (g_displayLevel==2)) g_displayLevel=1; + if (hasStdout && (g_displayLevel==2)) g_displayLevel=1; /* IO Stream/File */ + FIO_setHasStdoutOutput(fCtx, hasStdout); + FIO_setNbFilesTotal(fCtx, (int)filenames->tableSize); + FIO_determineHasStdinInput(fCtx, filenames); FIO_setNotificationLevel(g_displayLevel); FIO_setPatchFromMode(prefs, patchFromDictFileName != NULL); if (memLimit == 0) { @@ -1323,9 +1349,9 @@ int main(int const argCount, const char* argv[]) } if ((filenames->tableSize==1) && outFileName) - operationResult = FIO_compressFilename(prefs, outFileName, filenames->fileNames[0], dictFileName, cLevel, compressionParams); + operationResult = FIO_compressFilename(fCtx, prefs, outFileName, filenames->fileNames[0], dictFileName, cLevel, compressionParams); else - operationResult = FIO_compressMultipleFilenames(prefs, filenames->fileNames, (unsigned)filenames->tableSize, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams); + operationResult = FIO_compressMultipleFilenames(fCtx, prefs, filenames->fileNames, outMirroredDirName, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams); #else (void)contentSize; (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; (void)ZSTD_strategyMap; /* not used when ZSTD_NOCOMPRESS set */ DISPLAY("Compression not supported \n"); @@ -1333,9 +1359,9 @@ int main(int const argCount, const char* argv[]) } else { /* decompression or test */ #ifndef ZSTD_NODECOMPRESS if (filenames->tableSize == 1 && outFileName) { - operationResult = FIO_decompressFilename(prefs, outFileName, filenames->fileNames[0], dictFileName); + operationResult = FIO_decompressFilename(fCtx, prefs, outFileName, filenames->fileNames[0], dictFileName); } else { - operationResult = FIO_decompressMultipleFilenames(prefs, filenames->fileNames, (unsigned)filenames->tableSize, outDirName, outFileName, dictFileName); + operationResult = FIO_decompressMultipleFilenames(fCtx, prefs, filenames->fileNames, outMirroredDirName, outDirName, outFileName, dictFileName); } #else DISPLAY("Decompression not supported \n"); @@ -1344,6 +1370,7 @@ int main(int const argCount, const char* argv[]) _end: FIO_freePreferences(prefs); + FIO_freeContext(fCtx); if (main_pause) waitEnter(); UTIL_freeFileNamesTable(filenames); UTIL_freeFileNamesTable(file_of_names); |