diff options
Diffstat (limited to 'programs/fileio.c')
| -rw-r--r-- | programs/fileio.c | 535 |
1 files changed, 331 insertions, 204 deletions
diff --git a/programs/fileio.c b/programs/fileio.c index 9fb795ed3f27b..7ada5921b1231 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -24,13 +24,14 @@ * Includes ***************************************/ #include "platform.h" /* Large Files support, SET_BINARY_MODE */ -#include "util.h" /* UTIL_getFileSize, UTIL_isRegularFile */ +#include "util.h" /* UTIL_getFileSize, UTIL_isRegularFile, UTIL_isSameFile */ #include <stdio.h> /* fprintf, fopen, fread, _fileno, stdin, stdout */ #include <stdlib.h> /* malloc, free */ #include <string.h> /* strcmp, strlen */ #include <assert.h> #include <errno.h> /* errno */ #include <signal.h> +#include "timefn.h" /* UTIL_getTime, UTIL_clockSpanMicro */ #if defined (_MSC_VER) # include <sys/stat.h> @@ -79,23 +80,29 @@ /*-************************************* * Macros ***************************************/ + +struct FIO_display_prefs_s { + int displayLevel; /* 0 : no display; 1: errors; 2: + result + interaction + warnings; 3: + progression; 4: + information */ + U32 noProgress; +}; + +static FIO_display_prefs_t g_display_prefs = {2, 0}; + #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } } -static int g_displayLevel = 2; /* 0 : no display; 1: errors; 2: + result + interaction + warnings; 3: + progression; 4: + information */ -void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; } +#define DISPLAYLEVEL(l, ...) { if (g_display_prefs.displayLevel>=l) { DISPLAY(__VA_ARGS__); } } static const U64 g_refreshRate = SEC_TO_MICRO / 6; static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; -#define READY_FOR_UPDATE() (!g_noProgress && UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) +#define READY_FOR_UPDATE() (!g_display_prefs.noProgress && UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) #define DELAY_NEXT_UPDATE() { g_displayClock = UTIL_getTime(); } #define DISPLAYUPDATE(l, ...) { \ - if (g_displayLevel>=l && !g_noProgress) { \ - if (READY_FOR_UPDATE() || (g_displayLevel>=4)) { \ + if (g_display_prefs.displayLevel>=l && !g_display_prefs.noProgress) { \ + if (READY_FOR_UPDATE() || (g_display_prefs.displayLevel>=4)) { \ DELAY_NEXT_UPDATE(); \ DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stderr); \ + if (g_display_prefs.displayLevel>=4) fflush(stderr); \ } } } #undef MIN /* in case it would be already defined */ @@ -189,7 +196,7 @@ static void ABRThandler(int sig) { const char* name; void* addrlist[MAX_STACK_FRAMES]; char** symbollist; - U32 addrlen, i; + int addrlen, i; switch (sig) { case SIGABRT: name = "SIGABRT"; break; @@ -237,10 +244,13 @@ void FIO_addAbortHandler() ***************************************************************/ #if defined(_MSC_VER) && _MSC_VER >= 1400 # define LONG_SEEK _fseeki64 +# define LONG_TELL _ftelli64 #elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ # define LONG_SEEK fseeko +# define LONG_TELL ftello #elif defined(__MINGW32__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) && defined(__MSVCRT__) # define LONG_SEEK fseeko64 +# define LONG_TELL ftello64 #elif defined(_WIN32) && !defined(__DJGPP__) # include <windows.h> static int LONG_SEEK(FILE* file, __int64 offset, int origin) { @@ -261,101 +271,184 @@ void FIO_addAbortHandler() } #else # define LONG_SEEK fseek +# define LONG_TELL ftell #endif /*-************************************* -* Local Parameters - Not thread safe +* Parameters: Typedefs +***************************************/ + +struct FIO_prefs_s { + + /* Algorithm preferences */ + FIO_compressionType_t compressionType; + U32 sparseFileSupport; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */ + int dictIDFlag; + int checksumFlag; + int blockSize; + int overlapLog; + U32 adaptiveMode; + int rsyncable; + int minAdaptLevel; + int maxAdaptLevel; + int ldmFlag; + int ldmHashLog; + int ldmMinMatch; + int ldmBucketSizeLog; + int ldmHashRateLog; + ZSTD_literalCompressionMode_e literalCompressionMode; + + /* IO preferences */ + U32 removeSrcFile; + U32 overwrite; + + /* Computation resources preferences */ + unsigned memLimit; + int nbWorkers; +}; + + +/*-************************************* +* Parameters: Initialization +***************************************/ + +#define FIO_OVERLAP_LOG_NOTSET 9999 +#define FIO_LDM_PARAM_NOTSET 9999 + + +FIO_prefs_t* FIO_createPreferences(void) +{ + FIO_prefs_t* const ret = (FIO_prefs_t*)malloc(sizeof(FIO_prefs_t)); + if (!ret) EXM_THROW(21, "Allocation error : not enough memory"); + + ret->compressionType = FIO_zstdCompression; + ret->overwrite = 0; + ret->sparseFileSupport = ZSTD_SPARSE_DEFAULT; + ret->dictIDFlag = 1; + ret->checksumFlag = 1; + ret->removeSrcFile = 0; + ret->memLimit = 0; + ret->nbWorkers = 1; + ret->blockSize = 0; + ret->overlapLog = FIO_OVERLAP_LOG_NOTSET; + ret->adaptiveMode = 0; + ret->rsyncable = 0; + ret->minAdaptLevel = -50; /* initializing this value requires a constant, so ZSTD_minCLevel() doesn't work */ + ret->maxAdaptLevel = 22; /* initializing this value requires a constant, so ZSTD_maxCLevel() doesn't work */ + ret->ldmFlag = 0; + ret->ldmHashLog = 0; + ret->ldmMinMatch = 0; + ret->ldmBucketSizeLog = FIO_LDM_PARAM_NOTSET; + ret->ldmHashRateLog = FIO_LDM_PARAM_NOTSET; + ret->literalCompressionMode = ZSTD_lcm_auto; + return ret; +} + +void FIO_freePreferences(FIO_prefs_t* const prefs) +{ + free(prefs); +} + + +/*-************************************* +* Parameters: Display Options +***************************************/ + +void FIO_setNotificationLevel(int level) { g_display_prefs.displayLevel=level; } + +void FIO_setNoProgress(unsigned noProgress) { g_display_prefs.noProgress = noProgress; } + + +/*-************************************* +* Parameters: Setters ***************************************/ -static FIO_compressionType_t g_compressionType = FIO_zstdCompression; -void FIO_setCompressionType(FIO_compressionType_t compressionType) { g_compressionType = compressionType; } -static U32 g_overwrite = 0; -void FIO_overwriteMode(void) { g_overwrite = 1; } -static U32 g_sparseFileSupport = ZSTD_SPARSE_DEFAULT; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */ -void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport = sparse; } -static U32 g_dictIDFlag = 1; -void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; } -static U32 g_checksumFlag = 1; -void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; } -static U32 g_removeSrcFile = 0; -void FIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); } -static unsigned g_memLimit = 0; -void FIO_setMemLimit(unsigned memLimit) { g_memLimit = memLimit; } -static unsigned g_nbWorkers = 1; -void FIO_setNbWorkers(unsigned nbWorkers) { + +void FIO_setCompressionType(FIO_prefs_t* const prefs, FIO_compressionType_t compressionType) { prefs->compressionType = compressionType; } + +void FIO_overwriteMode(FIO_prefs_t* const prefs) { prefs->overwrite = 1; } + +void FIO_setSparseWrite(FIO_prefs_t* const prefs, unsigned sparse) { prefs->sparseFileSupport = sparse; } + +void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag) { prefs->dictIDFlag = dictIDFlag; } + +void FIO_setChecksumFlag(FIO_prefs_t* const prefs, int checksumFlag) { prefs->checksumFlag = checksumFlag; } + +void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag) { prefs->removeSrcFile = (flag>0); } + +void FIO_setMemLimit(FIO_prefs_t* const prefs, unsigned memLimit) { prefs->memLimit = memLimit; } + +void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers) { #ifndef ZSTD_MULTITHREAD if (nbWorkers > 0) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n"); #endif - g_nbWorkers = nbWorkers; + prefs->nbWorkers = nbWorkers; } -static U32 g_blockSize = 0; -void FIO_setBlockSize(unsigned blockSize) { - if (blockSize && g_nbWorkers==0) + +void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize) { + if (blockSize && prefs->nbWorkers==0) DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n"); - g_blockSize = blockSize; + prefs->blockSize = blockSize; } -#define FIO_OVERLAP_LOG_NOTSET 9999 -static unsigned g_overlapLog = FIO_OVERLAP_LOG_NOTSET; -void FIO_setOverlapLog(unsigned overlapLog){ - if (overlapLog && g_nbWorkers==0) + +void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog){ + if (overlapLog && prefs->nbWorkers==0) DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n"); - g_overlapLog = overlapLog; + prefs->overlapLog = overlapLog; } -static U32 g_adaptiveMode = 0; -void FIO_setAdaptiveMode(unsigned adapt) { - if ((adapt>0) && (g_nbWorkers==0)) + +void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, unsigned adapt) { + if ((adapt>0) && (prefs->nbWorkers==0)) EXM_THROW(1, "Adaptive mode is not compatible with single thread mode \n"); - g_adaptiveMode = adapt; + prefs->adaptiveMode = adapt; } -static U32 g_rsyncable = 0; -void FIO_setRsyncable(unsigned rsyncable) { - if ((rsyncable>0) && (g_nbWorkers==0)) + +void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable) { + if ((rsyncable>0) && (prefs->nbWorkers==0)) EXM_THROW(1, "Rsyncable mode is not compatible with single thread mode \n"); - g_rsyncable = rsyncable; + prefs->rsyncable = rsyncable; } -static int g_minAdaptLevel = -50; /* initializing this value requires a constant, so ZSTD_minCLevel() doesn't work */ -void FIO_setAdaptMin(int minCLevel) + +void FIO_setLiteralCompressionMode( + FIO_prefs_t* const prefs, + ZSTD_literalCompressionMode_e mode) { + prefs->literalCompressionMode = mode; +} + +void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel) { #ifndef ZSTD_NOCOMPRESS assert(minCLevel >= ZSTD_minCLevel()); #endif - g_minAdaptLevel = minCLevel; + prefs->minAdaptLevel = minCLevel; } -static int g_maxAdaptLevel = 22; /* initializing this value requires a constant, so ZSTD_maxCLevel() doesn't work */ -void FIO_setAdaptMax(int maxCLevel) + +void FIO_setAdaptMax(FIO_prefs_t* const prefs, int maxCLevel) { - g_maxAdaptLevel = maxCLevel; + prefs->maxAdaptLevel = maxCLevel; } -static U32 g_ldmFlag = 0; -void FIO_setLdmFlag(unsigned ldmFlag) { - g_ldmFlag = (ldmFlag>0); -} -static U32 g_ldmHashLog = 0; -void FIO_setLdmHashLog(unsigned ldmHashLog) { - g_ldmHashLog = ldmHashLog; -} -static U32 g_ldmMinMatch = 0; -void FIO_setLdmMinMatch(unsigned ldmMinMatch) { - g_ldmMinMatch = ldmMinMatch; +void FIO_setLdmFlag(FIO_prefs_t* const prefs, unsigned ldmFlag) { + prefs->ldmFlag = (ldmFlag>0); } -#define FIO_LDM_PARAM_NOTSET 9999 -static U32 g_ldmBucketSizeLog = FIO_LDM_PARAM_NOTSET; -void FIO_setLdmBucketSizeLog(unsigned ldmBucketSizeLog) { - g_ldmBucketSizeLog = ldmBucketSizeLog; +void FIO_setLdmHashLog(FIO_prefs_t* const prefs, int ldmHashLog) { + prefs->ldmHashLog = ldmHashLog; } -static U32 g_ldmHashRateLog = FIO_LDM_PARAM_NOTSET; -void FIO_setLdmHashRateLog(unsigned ldmHashRateLog) { - g_ldmHashRateLog = ldmHashRateLog; +void FIO_setLdmMinMatch(FIO_prefs_t* const prefs, int ldmMinMatch) { + prefs->ldmMinMatch = ldmMinMatch; } -static U32 g_noProgress = 0; -void FIO_setNoProgress(unsigned noProgress) { - g_noProgress = noProgress; + +void FIO_setLdmBucketSizeLog(FIO_prefs_t* const prefs, int ldmBucketSizeLog) { + prefs->ldmBucketSizeLog = ldmBucketSizeLog; } +void FIO_setLdmHashRateLog(FIO_prefs_t* const prefs, int ldmHashRateLog) { + prefs->ldmHashRateLog = ldmHashRateLog; +} + /*-************************************* * Functions @@ -410,45 +503,27 @@ static FILE* FIO_openSrcFile(const char* srcFileName) /** FIO_openDstFile() : * condition : `dstFileName` must be non-NULL. * @result : FILE* to `dstFileName`, or NULL if it fails */ -static FILE* FIO_openDstFile(const char* srcFileName, const char* dstFileName) +static FILE* FIO_openDstFile(FIO_prefs_t* const prefs, const char* srcFileName, const char* dstFileName) { assert(dstFileName != NULL); if (!strcmp (dstFileName, stdoutmark)) { DISPLAYLEVEL(4,"Using stdout for output \n"); SET_BINARY_MODE(stdout); - if (g_sparseFileSupport == 1) { - g_sparseFileSupport = 0; + if (prefs->sparseFileSupport == 1) { + prefs->sparseFileSupport = 0; DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n"); } return stdout; } - /* ensure dst is not the same file as src */ - if (srcFileName != NULL) { -#ifdef _MSC_VER - /* note : Visual does not support file identification by inode. - * The following work-around is limited to detecting exact name repetition only, - * aka `filename` is considered different from `subdir/../filename` */ - if (!strcmp(srcFileName, dstFileName)) { - DISPLAYLEVEL(1, "zstd: Refusing to open a output file which will overwrite the input file \n"); - return NULL; - } -#else - stat_t srcStat; - stat_t dstStat; - if (UTIL_getFileStat(srcFileName, &srcStat) - && UTIL_getFileStat(dstFileName, &dstStat)) { - if (srcStat.st_dev == dstStat.st_dev - && srcStat.st_ino == dstStat.st_ino) { - DISPLAYLEVEL(1, "zstd: Refusing to open a output file which will overwrite the input file \n"); - return NULL; - } - } -#endif + /* ensure dst is not the same as src */ + if (srcFileName != NULL && UTIL_isSameFile(srcFileName, dstFileName)) { + DISPLAYLEVEL(1, "zstd: Refusing to open an output file which will overwrite the input file \n"); + return NULL; } - if (g_sparseFileSupport == 1) { - g_sparseFileSupport = ZSTD_SPARSE_DEFAULT; + if (prefs->sparseFileSupport == 1) { + prefs->sparseFileSupport = ZSTD_SPARSE_DEFAULT; } if (UTIL_isRegularFile(dstFileName)) { @@ -460,8 +535,8 @@ static FILE* FIO_openDstFile(const char* srcFileName, const char* dstFileName) } if (fCheck != NULL) { /* dst file exists, authorization prompt */ fclose(fCheck); - if (!g_overwrite) { - if (g_displayLevel <= 1) { + if (!prefs->overwrite) { + if (g_display_prefs.displayLevel <= 1) { /* No interaction possible */ DISPLAY("zstd: %s already exists; not overwritten \n", dstFileName); @@ -536,10 +611,12 @@ typedef struct { size_t srcBufferSize; void* dstBuffer; size_t dstBufferSize; + const char* dictFileName; ZSTD_CStream* cctx; } cRess_t; -static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, +static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, + const char* dictFileName, int cLevel, U64 srcSize, ZSTD_compressionParameters comprParams) { cRess_t ress; @@ -562,43 +639,45 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, size_t const dictBuffSize = FIO_createDictBuffer(&dictBuffer, dictFileName); /* works with dictFileName==NULL */ if (dictFileName && (dictBuffer==NULL)) EXM_THROW(32, "allocation error : can't create dictBuffer"); + ress.dictFileName = dictFileName; - if (g_adaptiveMode && !g_ldmFlag && !comprParams.windowLog) + if (prefs->adaptiveMode && !prefs->ldmFlag && !comprParams.windowLog) comprParams.windowLog = ADAPT_WINDOWLOG_DEFAULT; CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_contentSizeFlag, 1) ); /* always enable content size when available (note: supposed to be default) */ - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_dictIDFlag, g_dictIDFlag) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_checksumFlag, g_checksumFlag) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_dictIDFlag, prefs->dictIDFlag) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_checksumFlag, prefs->checksumFlag) ); /* compression level */ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, cLevel) ); /* long distance matching */ - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_enableLongDistanceMatching, g_ldmFlag) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashLog, g_ldmHashLog) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmMinMatch, g_ldmMinMatch) ); - if (g_ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) { - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmBucketSizeLog, g_ldmBucketSizeLog) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_enableLongDistanceMatching, prefs->ldmFlag) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashLog, prefs->ldmHashLog) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmMinMatch, prefs->ldmMinMatch) ); + if (prefs->ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) { + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmBucketSizeLog, prefs->ldmBucketSizeLog) ); } - if (g_ldmHashRateLog != FIO_LDM_PARAM_NOTSET) { - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashRateLog, g_ldmHashRateLog) ); + if (prefs->ldmHashRateLog != FIO_LDM_PARAM_NOTSET) { + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashRateLog, prefs->ldmHashRateLog) ); } /* compression parameters */ - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_windowLog, comprParams.windowLog) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_chainLog, comprParams.chainLog) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_hashLog, comprParams.hashLog) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_searchLog, comprParams.searchLog) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_minMatch, comprParams.minMatch) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetLength, comprParams.targetLength) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_windowLog, (int)comprParams.windowLog) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_chainLog, (int)comprParams.chainLog) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_hashLog, (int)comprParams.hashLog) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_searchLog, (int)comprParams.searchLog) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_minMatch, (int)comprParams.minMatch) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetLength, (int)comprParams.targetLength) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_strategy, comprParams.strategy) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_literalCompressionMode, (int)prefs->literalCompressionMode) ); /* multi-threading */ #ifdef ZSTD_MULTITHREAD - DISPLAYLEVEL(5,"set nb workers = %u \n", g_nbWorkers); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_nbWorkers, g_nbWorkers) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, g_blockSize) ); - if (g_overlapLog != FIO_OVERLAP_LOG_NOTSET) { - DISPLAYLEVEL(3,"set overlapLog = %u \n", g_overlapLog); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_overlapLog, g_overlapLog) ); + DISPLAYLEVEL(5,"set nb workers = %u \n", prefs->nbWorkers); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_nbWorkers, prefs->nbWorkers) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, prefs->blockSize) ); + if (prefs->overlapLog != FIO_OVERLAP_LOG_NOTSET) { + DISPLAYLEVEL(3,"set overlapLog = %u \n", prefs->overlapLog); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_overlapLog, prefs->overlapLog) ); } - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_rsyncable, g_rsyncable) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_rsyncable, prefs->rsyncable) ); #endif /* dictionary */ CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* set the value temporarily for dictionary loading, to adapt compression parameters */ @@ -774,15 +853,19 @@ FIO_compressLzmaFrame(cRess_t* ress, #endif #ifdef ZSTD_LZ4COMPRESS + #if LZ4_VERSION_NUMBER <= 10600 #define LZ4F_blockLinked blockLinked #define LZ4F_max64KB max64KB #endif + static int FIO_LZ4_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } + static unsigned long long FIO_compressLz4Frame(cRess_t* ress, const char* srcFileName, U64 const srcFileSize, - int compressionLevel, U64* readsize) + int compressionLevel, int checksumFlag, + U64* readsize) { const size_t blockSize = FIO_LZ4_GetBlockSize_FromBlockId(LZ4F_max64KB); unsigned long long inFileSize = 0, outFileSize = 0; @@ -802,7 +885,7 @@ FIO_compressLz4Frame(cRess_t* ress, prefs.compressionLevel = compressionLevel; prefs.frameInfo.blockMode = LZ4F_blockLinked; prefs.frameInfo.blockSizeID = LZ4F_max64KB; - prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)g_checksumFlag; + prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)checksumFlag; #if LZ4_VERSION_NUMBER >= 10600 prefs.frameInfo.contentSize = (srcFileSize==UTIL_FILESIZE_UNKNOWN) ? 0 : srcFileSize; #endif @@ -876,7 +959,8 @@ FIO_compressLz4Frame(cRess_t* ress, static unsigned long long -FIO_compressZstdFrame(const cRess_t* ressPtr, +FIO_compressZstdFrame(FIO_prefs_t* const prefs, + const cRess_t* ressPtr, const char* srcFileName, U64 fileSize, int compressionLevel, U64* readsize) { @@ -947,7 +1031,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr, double const cShare = (double)zfp.produced / (zfp.consumed + !zfp.consumed/*avoid div0*/) * 100; /* display progress notifications */ - if (g_displayLevel >= 3) { + if (g_display_prefs.displayLevel >= 3) { DISPLAYUPDATE(3, "\r(L%i) Buffered :%4u MB - Consumed :%4u MB - Compressed :%4u MB => %.2f%% ", compressionLevel, (unsigned)((zfp.ingested - zfp.consumed) >> 20), @@ -963,7 +1047,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr, } /* adaptive mode : statistics measurement and speed correction */ - if (g_adaptiveMode) { + if (prefs->adaptiveMode) { /* check output speed */ if (zfp.currentJobID > 1) { /* only possible if nbWorkers >= 1 */ @@ -971,12 +1055,12 @@ FIO_compressZstdFrame(const cRess_t* ressPtr, unsigned long long newlyProduced = zfp.produced - previous_zfp_update.produced; unsigned long long newlyFlushed = zfp.flushed - previous_zfp_update.flushed; assert(zfp.produced >= previous_zfp_update.produced); - assert(g_nbWorkers >= 1); + assert(prefs->nbWorkers >= 1); /* test if compression is blocked * either because output is slow and all buffers are full * or because input is slow and no job can start while waiting for at least one buffer to be filled. - * note : excluse starting part, since currentJobID > 1 */ + * note : exclude starting part, since currentJobID > 1 */ if ( (zfp.consumed == previous_zfp_update.consumed) /* no data compressed : no data available, or no more buffer to compress to, OR compression is really slow (compression of a single block is slower than update rate)*/ && (zfp.nbActiveWorkers == 0) /* confirmed : no compression ongoing */ ) { @@ -1000,7 +1084,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr, DISPLAYLEVEL(6, "compression level adaptation check \n") /* check input speed */ - if (zfp.currentJobID > g_nbWorkers+1) { /* warm up period, to fill all workers */ + if (zfp.currentJobID > (unsigned)(prefs->nbWorkers+1)) { /* warm up period, to fill all workers */ if (inputBlocked <= 0) { DISPLAYLEVEL(6, "input is never blocked => input is slower than ingestion \n"); speedChange = slower; @@ -1032,14 +1116,14 @@ FIO_compressZstdFrame(const cRess_t* ressPtr, DISPLAYLEVEL(6, "slower speed , higher compression \n") compressionLevel ++; if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel(); - if (compressionLevel > g_maxAdaptLevel) compressionLevel = g_maxAdaptLevel; + if (compressionLevel > prefs->maxAdaptLevel) compressionLevel = prefs->maxAdaptLevel; compressionLevel += (compressionLevel == 0); /* skip 0 */ ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel); } if (speedChange == faster) { DISPLAYLEVEL(6, "faster speed , lighter compression \n") compressionLevel --; - if (compressionLevel < g_minAdaptLevel) compressionLevel = g_minAdaptLevel; + if (compressionLevel < prefs->minAdaptLevel) compressionLevel = prefs->minAdaptLevel; compressionLevel -= (compressionLevel == 0); /* skip 0 */ ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel); } @@ -1069,7 +1153,8 @@ FIO_compressZstdFrame(const cRess_t* ressPtr, * 1 : missing or pb opening srcFileName */ static int -FIO_compressFilename_internal(cRess_t ress, +FIO_compressFilename_internal(FIO_prefs_t* const prefs, + cRess_t ress, const char* dstFileName, const char* srcFileName, int compressionLevel) { @@ -1079,10 +1164,10 @@ FIO_compressFilename_internal(cRess_t ress, DISPLAYLEVEL(5, "%s: %u bytes \n", srcFileName, (unsigned)fileSize); /* compression format selection */ - switch (g_compressionType) { + switch (prefs->compressionType) { default: case FIO_zstdCompression: - compressedfilesize = FIO_compressZstdFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize); + compressedfilesize = FIO_compressZstdFrame(prefs, &ress, srcFileName, fileSize, compressionLevel, &readsize); break; case FIO_gzipCompression: @@ -1098,7 +1183,7 @@ FIO_compressFilename_internal(cRess_t ress, case FIO_xzCompression: case FIO_lzmaCompression: #ifdef ZSTD_LZMACOMPRESS - compressedfilesize = FIO_compressLzmaFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize, g_compressionType==FIO_lzmaCompression); + compressedfilesize = FIO_compressLzmaFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize, prefs->compressionType==FIO_lzmaCompression); #else (void)compressionLevel; EXM_THROW(20, "zstd: %s: file cannot be compressed as xz/lzma (zstd compiled without ZSTD_LZMACOMPRESS) -- ignored \n", @@ -1108,7 +1193,7 @@ FIO_compressFilename_internal(cRess_t ress, case FIO_lz4Compression: #ifdef ZSTD_LZ4COMPRESS - compressedfilesize = FIO_compressLz4Frame(&ress, srcFileName, fileSize, compressionLevel, &readsize); + compressedfilesize = FIO_compressLz4Frame(&ress, srcFileName, fileSize, compressionLevel, prefs->checksumFlag, &readsize); #else (void)compressionLevel; EXM_THROW(20, "zstd: %s: file cannot be compressed as lz4 (zstd compiled without ZSTD_LZ4COMPRESS) -- ignored \n", @@ -1138,7 +1223,8 @@ FIO_compressFilename_internal(cRess_t ress, * @return : 0 : compression completed correctly, * 1 : pb */ -static int FIO_compressFilename_dstFile(cRess_t ress, +static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs, + cRess_t ress, const char* dstFileName, const char* srcFileName, int compressionLevel) @@ -1153,7 +1239,7 @@ static int FIO_compressFilename_dstFile(cRess_t ress, if (ress.dstFile == NULL) { closeDstFile = 1; DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s", dstFileName); - ress.dstFile = FIO_openDstFile(srcFileName, dstFileName); + ress.dstFile = FIO_openDstFile(prefs, srcFileName, dstFileName); if (ress.dstFile==NULL) return 1; /* could not open dstFileName */ /* Must only be added after FIO_openDstFile() succeeds. * Otherwise we may delete the destination file if it already exists, @@ -1166,7 +1252,7 @@ static int FIO_compressFilename_dstFile(cRess_t ress, transfer_permissions = 1; } - result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, compressionLevel); + result = FIO_compressFilename_internal(prefs, ress, dstFileName, srcFileName, compressionLevel); if (closeDstFile) { FILE* const dstFile = ress.dstFile; @@ -1199,27 +1285,34 @@ static int FIO_compressFilename_dstFile(cRess_t ress, * 1 : missing or pb opening srcFileName */ static int -FIO_compressFilename_srcFile(cRess_t ress, +FIO_compressFilename_srcFile(FIO_prefs_t* const prefs, + cRess_t ress, const char* dstFileName, const char* srcFileName, int compressionLevel) { int result; - /* File check */ + /* ensure src is not a directory */ if (UTIL_isDirectory(srcFileName)) { DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); return 1; } + /* ensure src is not the same as dict (if present) */ + if (ress.dictFileName != NULL && UTIL_isSameFile(srcFileName, ress.dictFileName)) { + DISPLAYLEVEL(1, "zstd: cannot use %s as an input file and dictionary \n", srcFileName); + return 1; + } + ress.srcFile = FIO_openSrcFile(srcFileName); if (ress.srcFile == NULL) return 1; /* srcFile could not be opened */ - result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel); + result = FIO_compressFilename_dstFile(prefs, ress, dstFileName, srcFileName, compressionLevel); fclose(ress.srcFile); ress.srcFile = NULL; - if ( g_removeSrcFile /* --rm */ + if ( prefs->removeSrcFile /* --rm */ && result == 0 /* success */ && strcmp(srcFileName, stdinmark) /* exception : don't erase stdin */ ) { @@ -1234,7 +1327,8 @@ FIO_compressFilename_srcFile(cRess_t ress, } -int FIO_compressFilename(const char* dstFileName, const char* srcFileName, +int FIO_compressFilename(FIO_prefs_t* const prefs, + const char* dstFileName, const char* srcFileName, const char* dictFileName, int compressionLevel, ZSTD_compressionParameters comprParams) { @@ -1242,8 +1336,8 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, U64 const fileSize = UTIL_getFileSize(srcFileName); U64 const srcSize = (fileSize == UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : fileSize; - cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams); - int const result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, compressionLevel); + cRess_t const ress = FIO_createCResources(prefs, dictFileName, compressionLevel, srcSize, comprParams); + int const result = FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC; DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds); @@ -1288,7 +1382,8 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix) * into one destination (outFileName) * or into one file each (outFileName == NULL, but suffix != NULL). */ -int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles, +int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, + const char** inFileNamesTable, unsigned nbFiles, const char* outFileName, const char* suffix, const char* dictFileName, int compressionLevel, ZSTD_compressionParameters comprParams) @@ -1297,19 +1392,19 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile U64 const firstFileSize = UTIL_getFileSize(inFileNamesTable[0]); U64 const firstSrcSize = (firstFileSize == UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : firstFileSize; U64 const srcSize = (nbFiles != 1) ? ZSTD_CONTENTSIZE_UNKNOWN : firstSrcSize ; - cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams); + cRess_t ress = FIO_createCResources(prefs, dictFileName, compressionLevel, srcSize, comprParams); /* init */ assert(outFileName != NULL || suffix != NULL); if (outFileName != NULL) { /* output into a single destination (stdout typically) */ - ress.dstFile = FIO_openDstFile(NULL, outFileName); + ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName); if (ress.dstFile == NULL) { /* could not open outFileName */ error = 1; } else { unsigned u; for (u=0; u<nbFiles; u++) - error |= FIO_compressFilename_srcFile(ress, outFileName, inFileNamesTable[u], compressionLevel); + error |= FIO_compressFilename_srcFile(prefs, ress, outFileName, inFileNamesTable[u], compressionLevel); if (fclose(ress.dstFile)) EXM_THROW(29, "Write error (%s) : cannot properly close %s", strerror(errno), outFileName); @@ -1320,7 +1415,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile for (u=0; u<nbFiles; u++) { const char* const srcFileName = inFileNamesTable[u]; const char* const dstFileName = FIO_determineCompressedName(srcFileName, suffix); /* cannot fail */ - error |= FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, compressionLevel); + error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); } } FIO_freeCResources(ress); @@ -1346,7 +1441,7 @@ typedef struct { FILE* dstFile; } dRess_t; -static dRess_t FIO_createDResources(const char* dictFileName) +static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFileName) { dRess_t ress; memset(&ress, 0, sizeof(ress)); @@ -1355,7 +1450,7 @@ static dRess_t FIO_createDResources(const char* dictFileName) ress.dctx = ZSTD_createDStream(); if (ress.dctx==NULL) EXM_THROW(60, "Error: %s : can't create ZSTD_DStream", strerror(errno)); - CHECK( ZSTD_DCtx_setMaxWindowSize(ress.dctx, g_memLimit) ); + CHECK( ZSTD_DCtx_setMaxWindowSize(ress.dctx, prefs->memLimit) ); ress.srcBufferSize = ZSTD_DStreamInSize(); ress.srcBuffer = malloc(ress.srcBufferSize); ress.dstBufferSize = ZSTD_DStreamOutSize(); @@ -1383,7 +1478,7 @@ static void FIO_freeDResources(dRess_t ress) /** FIO_fwriteSparse() : * @return : storedSkips, to be provided to next call to FIO_fwriteSparse() of LZ4IO_fwriteSparseEnd() */ -static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips) +static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips) { const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */ size_t bufferSizeT = bufferSize / sizeof(size_t); @@ -1391,7 +1486,7 @@ static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSi const size_t* ptrT = bufferT; static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* 0-test re-attempted every 32 KB */ - if (!g_sparseFileSupport) { /* normal write */ + if (!prefs->sparseFileSupport) { /* normal write */ size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); if (sizeCheck != bufferSize) EXM_THROW(70, "Write error : %s (cannot write decoded block)", @@ -1444,7 +1539,7 @@ static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSi if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse"); storedSkips = 0; - { size_t const sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file); + { size_t const sizeCheck = fwrite(restPtr, 1, (size_t)(restEnd - restPtr), file); if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(75, "Write error : cannot write decoded end of block"); } } } } @@ -1452,10 +1547,12 @@ static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSi return storedSkips; } -static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips) +static void +FIO_fwriteSparseEnd(FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips) { if (storedSkips>0) { - assert(g_sparseFileSupport > 0); /* storedSkips>0 implies sparse support is enabled */ + assert(prefs->sparseFileSupport > 0); /* storedSkips>0 implies sparse support is enabled */ + (void)prefs; /* assert can be disabled, in which case prefs becomes unused */ if (LONG_SEEK(file, storedSkips-1, SEEK_CUR) != 0) EXM_THROW(69, "Final skip error (sparse file support)"); /* last zero must be explicitly written, @@ -1469,7 +1566,10 @@ static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips) /** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode @return : 0 (no error) */ -static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_t bufferSize, size_t alreadyLoaded) +static int FIO_passThrough(FIO_prefs_t* const prefs, + FILE* foutput, FILE* finput, + void* buffer, size_t bufferSize, + size_t alreadyLoaded) { size_t const blockSize = MIN(64 KB, bufferSize); size_t readFromInput = 1; @@ -1484,10 +1584,10 @@ static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_ while (readFromInput) { readFromInput = fread(buffer, 1, blockSize, finput); - storedSkips = FIO_fwriteSparse(foutput, buffer, readFromInput, storedSkips); + storedSkips = FIO_fwriteSparse(prefs, foutput, buffer, readFromInput, storedSkips); } - FIO_fwriteSparseEnd(foutput, storedSkips); + FIO_fwriteSparseEnd(prefs, foutput, storedSkips); return 0; } @@ -1506,7 +1606,7 @@ static unsigned FIO_highbit64(unsigned long long v) /* FIO_zstdErrorHelp() : * detailed error message when requested window size is too large */ -static void FIO_zstdErrorHelp(dRess_t* ress, size_t err, char const* srcFileName) +static void FIO_zstdErrorHelp(FIO_prefs_t* const prefs, dRess_t* ress, size_t err, char const* srcFileName) { ZSTD_frameHeader header; @@ -1519,9 +1619,9 @@ static void FIO_zstdErrorHelp(dRess_t* ress, size_t err, char const* srcFileName if (err == 0) { unsigned long long const windowSize = header.windowSize; unsigned const windowLog = FIO_highbit64(windowSize) + ((windowSize & (windowSize - 1)) != 0); - assert(g_memLimit > 0); + assert(prefs->memLimit > 0); DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u\n", - srcFileName, windowSize, g_memLimit); + srcFileName, windowSize, prefs->memLimit); if (windowLog <= ZSTD_WINDOWLOG_MAX) { unsigned const windowMB = (unsigned)((windowSize >> 20) + ((windowSize & ((1 MB) - 1)) != 0)); assert(windowSize < (U64)(1ULL << 52)); /* ensure now overflow for windowMB */ @@ -1538,7 +1638,9 @@ static void FIO_zstdErrorHelp(dRess_t* ress, size_t err, char const* srcFileName * @return : size of decoded zstd frame, or an error code */ #define FIO_ERROR_FRAME_DECODING ((unsigned long long)(-2)) -static unsigned long long FIO_decompressZstdFrame(dRess_t* ress, +static unsigned long long FIO_decompressZstdFrame( + FIO_prefs_t* const prefs, + dRess_t* ress, FILE* finput, const char* srcFileName, U64 alreadyDecoded) @@ -1567,12 +1669,12 @@ static unsigned long long FIO_decompressZstdFrame(dRess_t* ress, if (ZSTD_isError(readSizeHint)) { DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n", srcFileName, ZSTD_getErrorName(readSizeHint)); - FIO_zstdErrorHelp(ress, readSizeHint, srcFileName); + FIO_zstdErrorHelp(prefs, ress, readSizeHint, srcFileName); return FIO_ERROR_FRAME_DECODING; } /* Write block */ - storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, storedSkips); + storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, outBuff.pos, storedSkips); frameSize += outBuff.pos; DISPLAYUPDATE(2, "\r%-20.20s : %u MB... ", srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); @@ -1603,7 +1705,7 @@ static unsigned long long FIO_decompressZstdFrame(dRess_t* ress, ress->srcBufferLoaded += readSize; } } } - FIO_fwriteSparseEnd(ress->dstFile, storedSkips); + FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips); return frameSize; } @@ -1834,7 +1936,7 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress, * @return : 0 : OK * 1 : error */ -static int FIO_decompressFrames(dRess_t ress, FILE* srcFile, +static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* srcFile, const char* dstFileName, const char* srcFileName) { unsigned readSomething = 0; @@ -1862,7 +1964,7 @@ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile, return 1; } if (ZSTD_isFrame(buf, ress.srcBufferLoaded)) { - unsigned long long const frameSize = FIO_decompressZstdFrame(&ress, srcFile, srcFileName, filesize); + unsigned long long const frameSize = FIO_decompressZstdFrame(prefs, &ress, srcFile, srcFileName, filesize); if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; filesize += frameSize; } else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */ @@ -1893,9 +1995,11 @@ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile, DISPLAYLEVEL(1, "zstd: %s: lz4 file cannot be uncompressed (zstd compiled without HAVE_LZ4) -- ignored \n", srcFileName); return 1; #endif - } else if ((g_overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */ - return FIO_passThrough(ress.dstFile, srcFile, - ress.srcBuffer, ress.srcBufferSize, ress.srcBufferLoaded); + } else if ((prefs->overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */ + return FIO_passThrough(prefs, + ress.dstFile, srcFile, + ress.srcBuffer, ress.srcBufferSize, + ress.srcBufferLoaded); } else { DISPLAYLEVEL(1, "zstd: %s: unsupported format \n", srcFileName); return 1; @@ -1915,7 +2019,8 @@ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile, @return : 0 : OK 1 : operation aborted */ -static int FIO_decompressDstFile(dRess_t ress, FILE* srcFile, +static int FIO_decompressDstFile(FIO_prefs_t* const prefs, + dRess_t ress, FILE* srcFile, const char* dstFileName, const char* srcFileName) { int result; @@ -1926,7 +2031,7 @@ static int FIO_decompressDstFile(dRess_t ress, FILE* srcFile, if (ress.dstFile == NULL) { releaseDstFile = 1; - ress.dstFile = FIO_openDstFile(srcFileName, dstFileName); + ress.dstFile = FIO_openDstFile(prefs, srcFileName, dstFileName); if (ress.dstFile==0) return 1; /* Must only be added after FIO_openDstFile() succeeds. @@ -1941,7 +2046,7 @@ static int FIO_decompressDstFile(dRess_t ress, FILE* srcFile, } - result = FIO_decompressFrames(ress, srcFile, dstFileName, srcFileName); + result = FIO_decompressFrames(prefs, ress, srcFile, dstFileName, srcFileName); if (releaseDstFile) { FILE* const dstFile = ress.dstFile; @@ -1974,7 +2079,7 @@ static int FIO_decompressDstFile(dRess_t ress, FILE* srcFile, @return : 0 : OK 1 : error */ -static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const char* srcFileName) +static int FIO_decompressSrcFile(FIO_prefs_t* const prefs, dRess_t ress, const char* dstFileName, const char* srcFileName) { FILE* srcFile; int result; @@ -1988,14 +2093,14 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch if (srcFile==NULL) return 1; ress.srcBufferLoaded = 0; - result = FIO_decompressDstFile(ress, srcFile, dstFileName, srcFileName); + result = FIO_decompressDstFile(prefs, ress, srcFile, dstFileName, srcFileName); /* Close file */ if (fclose(srcFile)) { DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); /* error should not happen */ return 1; } - if ( g_removeSrcFile /* --rm */ + if ( prefs->removeSrcFile /* --rm */ && (result==0) /* decompression successful */ && strcmp(srcFileName, stdinmark) ) /* not stdin */ { /* We must clear the handler, since after this point calling it would @@ -2012,12 +2117,13 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch -int FIO_decompressFilename(const char* dstFileName, const char* srcFileName, +int FIO_decompressFilename(FIO_prefs_t* const prefs, + const char* dstFileName, const char* srcFileName, const char* dictFileName) { - dRess_t const ress = FIO_createDResources(dictFileName); + dRess_t const ress = FIO_createDResources(prefs, dictFileName); - int const decodingError = FIO_decompressSrcFile(ress, dstFileName, srcFileName); + int const decodingError = FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName); FIO_freeDResources(ress); return decodingError; @@ -2094,19 +2200,20 @@ FIO_determineDstName(const char* srcFileName) int -FIO_decompressMultipleFilenames(const char* srcNamesTable[], unsigned nbFiles, +FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, + const char* srcNamesTable[], unsigned nbFiles, const char* outFileName, const char* dictFileName) { int error = 0; - dRess_t ress = FIO_createDResources(dictFileName); + dRess_t ress = FIO_createDResources(prefs, dictFileName); if (outFileName) { unsigned u; - ress.dstFile = FIO_openDstFile(NULL, outFileName); + ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName); if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", outFileName); for (u=0; u<nbFiles; u++) - error |= FIO_decompressSrcFile(ress, outFileName, srcNamesTable[u]); + error |= FIO_decompressSrcFile(prefs, ress, outFileName, srcNamesTable[u]); if (fclose(ress.dstFile)) EXM_THROW(72, "Write error : %s : cannot properly close output file", strerror(errno)); @@ -2117,7 +2224,7 @@ FIO_decompressMultipleFilenames(const char* srcNamesTable[], unsigned nbFiles, const char* const dstFileName = FIO_determineDstName(srcFileName); if (dstFileName == NULL) { error=1; continue; } - error |= FIO_decompressSrcFile(ress, dstFileName, srcFileName); + error |= FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName); } } @@ -2142,7 +2249,13 @@ typedef struct { U32 nbFiles; } fileInfo_t; -typedef enum { info_success=0, info_frame_error=1, info_not_zstd=2, info_file_error=3 } InfoError; +typedef enum { + info_success=0, + info_frame_error=1, + info_not_zstd=2, + info_file_error=3, + info_truncated_input=4, +} InfoError; #define ERROR_IF(c,n,...) { \ if (c) { \ @@ -2164,6 +2277,12 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile) && (numBytesRead == 0) && (info->compressedSize > 0) && (info->compressedSize != UTIL_FILESIZE_UNKNOWN) ) { + unsigned long long file_position = (unsigned long long) LONG_TELL(srcFile); + unsigned long long file_size = (unsigned long long) info->compressedSize; + ERROR_IF(file_position != file_size, info_truncated_input, + "Error: seeked to position %llu, which is beyond file size of %llu\n", + file_position, + file_size); break; /* correct end of file => success */ } ERROR_IF(feof(srcFile), info_not_zstd, "Error: reached end of file with incomplete frame"); @@ -2332,20 +2451,28 @@ FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel) fileInfo_t info; memset(&info, 0, sizeof(info)); { InfoError const error = getFileInfo(&info, inFileName); - if (error == info_frame_error) { - /* display error, but provide output */ - DISPLAYLEVEL(1, "Error while parsing %s \n", inFileName); - } - else if (error == info_not_zstd) { - DISPLAYOUT("File %s not compressed by zstd \n", inFileName); - if (displayLevel > 2) DISPLAYOUT("\n"); - return 1; - } - else if (error == info_file_error) { - /* error occurred while opening the file */ - if (displayLevel > 2) DISPLAYOUT("\n"); - return 1; + switch (error) { + case info_frame_error: + /* display error, but provide output */ + DISPLAYLEVEL(1, "Error while parsing \"%s\" \n", inFileName); + break; + case info_not_zstd: + DISPLAYOUT("File \"%s\" not compressed by zstd \n", inFileName); + if (displayLevel > 2) DISPLAYOUT("\n"); + return 1; + case info_file_error: + /* error occurred while opening the file */ + if (displayLevel > 2) DISPLAYOUT("\n"); + return 1; + case info_truncated_input: + DISPLAYOUT("File \"%s\" is truncated \n", inFileName); + if (displayLevel > 2) DISPLAYOUT("\n"); + return 1; + case info_success: + default: + break; } + displayInfo(inFileName, &info, displayLevel); *total = FIO_addFInfo(*total, info); assert(error == info_success || error == info_frame_error); |