diff options
Diffstat (limited to 'programs/zstdcli.c')
| -rw-r--r-- | programs/zstdcli.c | 280 |
1 files changed, 229 insertions, 51 deletions
diff --git a/programs/zstdcli.c b/programs/zstdcli.c index c35de7ccfbbd3..1545d1cac5790 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -32,13 +32,13 @@ #include <errno.h> /* errno */ #include "fileio.h" /* stdinmark, stdoutmark, ZSTD_EXTENSION */ #ifndef ZSTD_NOBENCH -# include "bench.h" /* BMK_benchFiles, BMK_SetNbSeconds */ +# include "bench.h" /* BMK_benchFiles */ #endif #ifndef ZSTD_NODICT # include "dibio.h" /* ZDICT_cover_params_t, DiB_trainFromFiles() */ #endif -#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */ -#include "zstd.h" /* ZSTD_VERSION_STRING */ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_minCLevel */ +#include "zstd.h" /* ZSTD_VERSION_STRING, ZSTD_maxCLevel */ /*-************************************ @@ -85,6 +85,10 @@ static U32 g_ldmHashEveryLog = LDM_PARAM_DEFAULT; static U32 g_ldmBucketSizeLog = LDM_PARAM_DEFAULT; +#define DEFAULT_ACCEL 1 + +typedef enum { cover, fastCover, legacy } dictType; + /*-************************************ * Display Macros **************************************/ @@ -135,6 +139,7 @@ static int usage_advanced(const char* programName) DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel()); DISPLAY( "--long[=#]: enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog); DISPLAY( "--fast[=#]: switch to ultra fast compression level (default: %u)\n", 1); + DISPLAY( "--adapt : dynamically adapt compression level to I/O conditions \n"); #ifdef ZSTD_MULTITHREAD DISPLAY( " -T# : spawns # compression threads (default: 1, 0==# cores) \n"); DISPLAY( " -B# : select size of each job (default: 0==automatic) \n"); @@ -145,6 +150,7 @@ static int usage_advanced(const char* programName) #ifdef UTIL_HAS_CREATEFILELIST DISPLAY( " -r : operate recursively on directories \n"); #endif + DISPLAY( "--format=zstd : compress files to the .zstd format (default) \n"); #ifdef ZSTD_GZCOMPRESS DISPLAY( "--format=gzip : compress files to the .gz format \n"); #endif @@ -169,7 +175,8 @@ static int usage_advanced(const char* programName) DISPLAY( "\n"); DISPLAY( "Dictionary builder : \n"); DISPLAY( "--train ## : create a dictionary from a training set of files \n"); - DISPLAY( "--train-cover[=k=#,d=#,steps=#] : use the cover algorithm with optional args\n"); + DISPLAY( "--train-cover[=k=#,d=#,steps=#,split=#] : use the cover algorithm with optional args\n"); + DISPLAY( "--train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#] : use the fast cover algorithm with optional args\n"); DISPLAY( "--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: %u)\n", g_defaultSelectivityLevel); DISPLAY( " -o file : `file` is dictionary name (default: %s) \n", g_defaultDictName); DISPLAY( "--maxdict=# : limit dictionary to specified size (default: %u) \n", g_defaultMaxDictSize); @@ -219,20 +226,34 @@ static int exeNameMatch(const char* exeName, const char* test) (exeName[strlen(test)] == '\0' || exeName[strlen(test)] == '.'); } +static void errorOut(const char* msg) +{ + DISPLAY("%s \n", msg); exit(1); +} + /*! readU32FromChar() : * @return : unsigned integer value read from input in `char` format. * allows and interprets K, KB, KiB, M, MB and MiB suffix. * Will also modify `*stringPtr`, advancing it to position where it stopped reading. - * Note : function result can overflow if digit string > MAX_UINT */ + * Note : function will exit() program if digit sequence overflows */ static unsigned readU32FromChar(const char** stringPtr) { + const char errorMsg[] = "error: numeric value too large"; unsigned result = 0; - while ((**stringPtr >='0') && (**stringPtr <='9')) + while ((**stringPtr >='0') && (**stringPtr <='9')) { + unsigned const max = (((unsigned)(-1)) / 10) - 1; + if (result > max) errorOut(errorMsg); result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + } if ((**stringPtr=='K') || (**stringPtr=='M')) { + unsigned const maxK = ((unsigned)(-1)) >> 10; + if (result > maxK) errorOut(errorMsg); result <<= 10; - if (**stringPtr=='M') result <<= 10; - (*stringPtr)++ ; + if (**stringPtr=='M') { + if (result > maxK) errorOut(errorMsg); + result <<= 10; + } + (*stringPtr)++; /* skip `K` or `M` */ if (**stringPtr=='i') (*stringPtr)++; if (**stringPtr=='B') (*stringPtr)++; } @@ -267,10 +288,42 @@ static unsigned parseCoverParameters(const char* stringPtr, ZDICT_cover_params_t if (longCommandWArg(&stringPtr, "k=")) { params->k = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "d=")) { params->d = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "steps=")) { params->steps = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "split=")) { + unsigned splitPercentage = readU32FromChar(&stringPtr); + params->splitPoint = (double)splitPercentage / 100.0; + if (stringPtr[0]==',') { stringPtr++; continue; } else break; + } return 0; } if (stringPtr[0] != 0) return 0; - DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nsteps=%u\n", params->k, params->d, params->steps); + DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nsteps=%u\nsplit=%u\n", params->k, params->d, params->steps, (unsigned)(params->splitPoint * 100)); + return 1; +} + +/** + * parseFastCoverParameters() : + * reads fastcover parameters from *stringPtr (e.g. "--train-fastcover=k=48,d=8,f=20,steps=32,accel=2") into *params + * @return 1 means that fastcover parameters were correct + * @return 0 in case of malformed parameters + */ +static unsigned parseFastCoverParameters(const char* stringPtr, ZDICT_fastCover_params_t* params) +{ + memset(params, 0, sizeof(*params)); + for (; ;) { + if (longCommandWArg(&stringPtr, "k=")) { params->k = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "d=")) { params->d = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "f=")) { params->f = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "steps=")) { params->steps = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "accel=")) { params->accel = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "split=")) { + unsigned splitPercentage = readU32FromChar(&stringPtr); + params->splitPoint = (double)splitPercentage / 100.0; + if (stringPtr[0]==',') { stringPtr++; continue; } else break; + } + return 0; + } + if (stringPtr[0] != 0) return 0; + DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\n", params->k, params->d, params->f, params->steps, (unsigned)(params->splitPoint * 100), params->accel); return 1; } @@ -295,11 +348,48 @@ static ZDICT_cover_params_t defaultCoverParams(void) memset(¶ms, 0, sizeof(params)); params.d = 8; params.steps = 4; + params.splitPoint = 1.0; + return params; +} + +static ZDICT_fastCover_params_t defaultFastCoverParams(void) +{ + ZDICT_fastCover_params_t params; + memset(¶ms, 0, sizeof(params)); + params.d = 8; + params.f = 20; + params.steps = 4; + params.splitPoint = 0.75; /* different from default splitPoint of cover */ + params.accel = DEFAULT_ACCEL; return params; } #endif +/** parseAdaptParameters() : + * reads adapt parameters from *stringPtr (e.g. "--zstd=min=1,max=19) and store them into adaptMinPtr and adaptMaxPtr. + * Both adaptMinPtr and adaptMaxPtr must be already allocated and correctly initialized. + * There is no guarantee that any of these values will be updated. + * @return 1 means that parsing was successful, + * @return 0 in case of malformed parameters + */ +static unsigned parseAdaptParameters(const char* stringPtr, int* adaptMinPtr, int* adaptMaxPtr) +{ + for ( ; ;) { + if (longCommandWArg(&stringPtr, "min=")) { *adaptMinPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "max=")) { *adaptMaxPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + DISPLAYLEVEL(4, "invalid compression parameter \n"); + return 0; + } + if (stringPtr[0] != 0) return 0; /* check the end of string */ + if (*adaptMinPtr > *adaptMaxPtr) { + DISPLAYLEVEL(4, "incoherent adaptation limits \n"); + return 0; + } + return 1; +} + + /** parseCompressionParameters() : * reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6") into *params * @return 1 means that compression parameters were correct @@ -364,6 +454,15 @@ typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom #define CLEAN_RETURN(i) { operationResult = (i); goto _end; } +#ifdef ZSTD_NOCOMPRESS +/* symbols from compression library are not defined and should not be invoked */ +# define MINCLEVEL -50 +# define MAXCLEVEL 22 +#else +# define MINCLEVEL ZSTD_minCLevel() +# define MAXCLEVEL ZSTD_maxCLevel() +#endif + int main(int argCount, const char* argv[]) { int argNb, @@ -373,6 +472,9 @@ int main(int argCount, const char* argv[]) ldmFlag = 0, main_pause = 0, nbWorkers = 0, + adapt = 0, + adaptMin = MINCLEVEL, + adaptMax = MAXCLEVEL, nextArgumentIsOutFileName = 0, nextArgumentIsMaxDict = 0, nextArgumentIsDictID = 0, @@ -383,6 +485,7 @@ int main(int argCount, const char* argv[]) setRealTimePrio = 0, singleThread = 0, ultra=0; + double compressibility = 0.5; unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */ size_t blockSize = 0; zstd_operation_mode operation = zom_compress; @@ -408,14 +511,16 @@ int main(int argCount, const char* argv[]) #endif #ifndef ZSTD_NODICT ZDICT_cover_params_t coverParams = defaultCoverParams(); - int cover = 1; + ZDICT_fastCover_params_t fastCoverParams = defaultFastCoverParams(); + dictType dict = fastCover; +#endif +#ifndef ZSTD_NOBENCH + BMK_advancedParams_t benchParams = BMK_initAdvancedParams(); #endif /* init */ (void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */ - (void)dictCLevel; (void)dictSelect; (void)dictID; (void)maxDictSize; /* not used when ZSTD_NODICT set */ - (void)ultra; (void)cLevel; (void)ldmFlag; /* not used when ZSTD_NOCOMPRESS set */ (void)memLimit; /* not used when ZSTD_NODECOMPRESS set */ if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); } filenameTable[0] = stdinmark; @@ -426,7 +531,7 @@ int main(int argCount, const char* argv[]) #endif /* preset behaviors */ - if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0; + if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0; if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress; if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* supports multiple formats */ if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* behave like zcat, also supports multiple formats */ @@ -441,6 +546,9 @@ int main(int argCount, const char* argv[]) if (exeNameMatch(programName, ZSTD_UNLZ4)) { operation=zom_decompress; FIO_setCompressionType(FIO_lz4Compression); } /* behave like unlz4, also supports multiple formats */ memset(&compressionParams, 0, sizeof(compressionParams)); + /* init crash handler */ + FIO_addAbortHandler(); + /* command switches */ for (argNb=1; argNb<argCount; argNb++) { const char* argument = argv[argNb]; @@ -478,14 +586,17 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; } if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; } if (!strcmp(argument, "--test")) { operation=zom_test; continue; } - if (!strcmp(argument, "--train")) { operation=zom_train; outFileName=g_defaultDictName; continue; } + if (!strcmp(argument, "--train")) { operation=zom_train; if (outFileName==NULL) outFileName=g_defaultDictName; continue; } if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; lastCommand=1; continue; } /* kept available for compatibility with old syntax ; will be removed one day */ if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; lastCommand=1; continue; } /* kept available for compatibility with old syntax ; will be removed one day */ if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; } if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; } + if (!strcmp(argument, "--adapt")) { adapt = 1; continue; } + if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) CLEAN_RETURN(badusage(programName)); continue; } if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; } + if (!strcmp(argument, "--format=zstd")) { suffix = ZSTD_EXTENSION; FIO_setCompressionType(FIO_zstdCompression); continue; } #ifdef ZSTD_GZCOMPRESS if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); continue; } #endif @@ -501,18 +612,31 @@ int main(int argCount, const char* argv[]) #ifndef ZSTD_NODICT if (longCommandWArg(&argument, "--train-cover")) { operation = zom_train; - outFileName = g_defaultDictName; - cover = 1; + if (outFileName == NULL) + outFileName = g_defaultDictName; + dict = cover; /* Allow optional arguments following an = */ if (*argument == 0) { memset(&coverParams, 0, sizeof(coverParams)); } else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); } else if (!parseCoverParameters(argument, &coverParams)) { CLEAN_RETURN(badusage(programName)); } continue; } + if (longCommandWArg(&argument, "--train-fastcover")) { + operation = zom_train; + if (outFileName == NULL) + outFileName = g_defaultDictName; + dict = fastCover; + /* Allow optional arguments following an = */ + if (*argument == 0) { memset(&fastCoverParams, 0, sizeof(fastCoverParams)); } + else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); } + else if (!parseFastCoverParameters(argument, &fastCoverParams)) { CLEAN_RETURN(badusage(programName)); } + continue; + } if (longCommandWArg(&argument, "--train-legacy")) { operation = zom_train; - outFileName = g_defaultDictName; - cover = 0; + if (outFileName == NULL) + outFileName = g_defaultDictName; + dict = legacy; /* Allow optional arguments following an = */ if (*argument == 0) { continue; } else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); } @@ -544,13 +668,20 @@ int main(int argCount, const char* argv[]) compressionParams.windowLog = ldmWindowLog; continue; } +#ifndef ZSTD_NOCOMPRESS /* linking ZSTD_minCLevel() requires compression support */ if (longCommandWArg(&argument, "--fast")) { - /* Parse optional window log */ + /* Parse optional acceleration factor */ if (*argument == '=') { + U32 const maxFast = (U32)-ZSTD_minCLevel(); U32 fastLevel; ++argument; fastLevel = readU32FromChar(&argument); - if (fastLevel) cLevel = - (int)fastLevel; + if (fastLevel > maxFast) fastLevel = maxFast; + if (fastLevel) { + dictCLevel = cLevel = -(int)fastLevel; + } else { + CLEAN_RETURN(badusage(programName)); + } } else if (*argument != 0) { /* Invalid character following --fast */ CLEAN_RETURN(badusage(programName)); @@ -559,6 +690,7 @@ int main(int argCount, const char* argv[]) } continue; } +#endif /* fall-through, will trigger bad_usage() later on */ } @@ -589,7 +721,7 @@ int main(int argCount, const char* argv[]) /* Decoding */ case 'd': #ifndef ZSTD_NOBENCH - BMK_setDecodeOnlyMode(1); + benchParams.mode = BMK_decodeOnly; if (operation==zom_bench) { argument++; break; } /* benchmark decode (hidden option) */ #endif operation=zom_decompress; argument++; break; @@ -682,11 +814,19 @@ int main(int argCount, const char* argv[]) case 'p': argument++; #ifndef ZSTD_NOBENCH if ((*argument>='0') && (*argument<='9')) { - BMK_setAdditionalParam(readU32FromChar(&argument)); + benchParams.additionalParam = (int)readU32FromChar(&argument); } else #endif main_pause=1; break; + + /* Select compressibility of synthetic sample */ + case 'P': + { argument++; + compressibility = (double)readU32FromChar(&argument) / 100; + } + break; + /* unknown command */ default : CLEAN_RETURN(badusage(programName)); } @@ -743,8 +883,11 @@ int main(int argCount, const char* argv[]) nbWorkers = UTIL_countPhysicalCores(); DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers); } +#else + (void)singleThread; (void)nbWorkers; #endif +#ifdef UTIL_HAS_CREATEFILELIST g_utilDisplayLevel = g_displayLevel; if (!followLinks) { unsigned u; @@ -757,7 +900,6 @@ int main(int argCount, const char* argv[]) } filenameIdx = fileNamesNb; } -#ifdef UTIL_HAS_CREATEFILELIST if (recursive) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */ extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb, followLinks); if (extendedFileList) { @@ -768,6 +910,8 @@ int main(int argCount, const char* argv[]) filenameIdx = fileNamesNb; } } +#else + (void)followLinks; #endif if (operation == zom_list) { @@ -783,24 +927,48 @@ int main(int argCount, const char* argv[]) /* Check if benchmark is selected */ if (operation==zom_bench) { #ifndef ZSTD_NOBENCH - BMK_setNotificationLevel(g_displayLevel); - BMK_setSeparateFiles(separateFiles); - BMK_setBlockSize(blockSize); - BMK_setNbWorkers(nbWorkers); - BMK_setRealTime(setRealTimePrio); - BMK_setNbSeconds(bench_nbSeconds); - BMK_setLdmFlag(ldmFlag); - BMK_setLdmMinMatch(g_ldmMinMatch); - BMK_setLdmHashLog(g_ldmHashLog); + benchParams.blockSize = blockSize; + benchParams.nbWorkers = nbWorkers; + benchParams.realTime = setRealTimePrio; + benchParams.nbSeconds = bench_nbSeconds; + benchParams.ldmFlag = ldmFlag; + benchParams.ldmMinMatch = g_ldmMinMatch; + benchParams.ldmHashLog = g_ldmHashLog; if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) { - BMK_setLdmBucketSizeLog(g_ldmBucketSizeLog); + benchParams.ldmBucketSizeLog = g_ldmBucketSizeLog; } if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) { - BMK_setLdmHashEveryLog(g_ldmHashEveryLog); + benchParams.ldmHashEveryLog = g_ldmHashEveryLog; + } + + if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel(); + if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel(); + if (cLevelLast < cLevel) cLevelLast = cLevel; + if (cLevelLast > cLevel) + DISPLAYLEVEL(3, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); + if(filenameIdx) { + if(separateFiles) { + unsigned i; + for(i = 0; i < filenameIdx; i++) { + int c; + DISPLAYLEVEL(3, "Benchmarking %s \n", filenameTable[i]); + for(c = cLevel; c <= cLevelLast; c++) { + BMK_benchFilesAdvanced(&filenameTable[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams); + } + } + } else { + for(; cLevel <= cLevelLast; cLevel++) { + BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams); + } + } + } else { + for(; cLevel <= cLevelLast; cLevel++) { + BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &benchParams); + } } - BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams); + #else - (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; + (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; (void)compressibility; #endif goto _end; } @@ -812,18 +980,27 @@ int main(int argCount, const char* argv[]) zParams.compressionLevel = dictCLevel; zParams.notificationLevel = g_displayLevel; zParams.dictID = dictID; - if (cover) { + if (dict == cover) { int const optimize = !coverParams.k || !coverParams.d; coverParams.nbThreads = nbWorkers; coverParams.zParams = zParams; - operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, optimize); + operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, NULL, optimize); + } else if (dict == fastCover) { + int const optimize = !fastCoverParams.k || !fastCoverParams.d; + fastCoverParams.nbThreads = nbWorkers; + fastCoverParams.zParams = zParams; + operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, NULL, &fastCoverParams, optimize); } else { ZDICT_legacy_params_t dictParams; memset(&dictParams, 0, sizeof(dictParams)); dictParams.selectivityLevel = dictSelect; dictParams.zParams = zParams; - operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, &dictParams, NULL, 0); + operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, &dictParams, NULL, NULL, 0); } +#else + (void)dictCLevel; (void)dictSelect; (void)dictID; (void)maxDictSize; /* not used when ZSTD_NODICT set */ + DISPLAYLEVEL(1, "training mode not available \n"); + operationResult = 1; #endif goto _end; } @@ -866,24 +1043,25 @@ int main(int argCount, const char* argv[]) #ifndef ZSTD_NOCOMPRESS FIO_setNbWorkers(nbWorkers); FIO_setBlockSize((U32)blockSize); + if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(g_overlapLog); FIO_setLdmFlag(ldmFlag); FIO_setLdmHashLog(g_ldmHashLog); FIO_setLdmMinMatch(g_ldmMinMatch); - if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) { - FIO_setLdmBucketSizeLog(g_ldmBucketSizeLog); - } - if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) { - FIO_setLdmHashEveryLog(g_ldmHashEveryLog); - } + if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(g_ldmBucketSizeLog); + if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) FIO_setLdmHashEveryLog(g_ldmHashEveryLog); + FIO_setAdaptiveMode(adapt); + FIO_setAdaptMin(adaptMin); + FIO_setAdaptMax(adaptMax); + if (adaptMin > cLevel) cLevel = adaptMin; + if (adaptMax < cLevel) cLevel = adaptMax; - if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(g_overlapLog); if ((filenameIdx==1) && outFileName) - operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, &compressionParams); + operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, compressionParams); else - operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, &compressionParams); + operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams); #else - (void)suffix; - DISPLAY("Compression not supported\n"); + (void)suffix; (void)adapt; (void)ultra; (void)cLevel; (void)ldmFlag; /* not used when ZSTD_NOCOMPRESS set */ + DISPLAY("Compression not supported \n"); #endif } else { /* decompression or test */ #ifndef ZSTD_NODECOMPRESS @@ -900,7 +1078,7 @@ int main(int argCount, const char* argv[]) else operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, outFileName, dictFileName); #else - DISPLAY("Decompression not supported\n"); + DISPLAY("Decompression not supported \n"); #endif } |