summaryrefslogtreecommitdiff
path: root/programs
diff options
context:
space:
mode:
Diffstat (limited to 'programs')
-rw-r--r--programs/Makefile45
-rw-r--r--programs/README.md5
-rw-r--r--programs/benchfn.c263
-rw-r--r--programs/benchfn.h167
-rw-r--r--programs/benchzstd.c (renamed from programs/bench.c)420
-rw-r--r--programs/benchzstd.h (renamed from programs/bench.h)119
-rw-r--r--programs/datagen.c14
-rw-r--r--programs/dibio.c6
-rw-r--r--programs/fileio.c260
-rw-r--r--programs/fileio.h4
-rw-r--r--programs/platform.h1
-rw-r--r--programs/util.c673
-rw-r--r--programs/util.h631
-rw-r--r--programs/zstd.134
-rw-r--r--programs/zstd.1.md52
-rw-r--r--programs/zstdcli.c100
-rwxr-xr-xprograms/zstdgrep141
-rw-r--r--programs/zstdgrep.12
-rw-r--r--programs/zstdless.12
19 files changed, 1659 insertions, 1280 deletions
diff --git a/programs/Makefile b/programs/Makefile
index 32dbc67eff5c5..d1910fbb4b79f 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -29,7 +29,12 @@ LIBVER := $(shell echo $(LIBVER_SCRIPT))
ZSTD_VERSION = $(LIBVER)
-GREP = grep --color=never
+HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
+GREP_OPTIONS ?=
+ifeq ($HAVE_COLORNEVER, 1)
+GREP_OPTIONS += --color=never
+endif
+GREP = grep $(GREP_OPTIONS)
ifeq ($(shell $(CC) -v 2>&1 | $(GREP) -c "gcc version "), 1)
ALIGN_LOOP = -falign-loops=32
@@ -48,7 +53,7 @@ DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
- -Wredundant-decls -Wmissing-prototypes
+ -Wredundant-decls -Wmissing-prototypes -Wc++-compat
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
@@ -91,7 +96,7 @@ VOID = /dev/null
# thread detection
NO_THREAD_MSG := ==> no threads, building without multithreading support
-HAVE_PTHREAD := $(shell printf '\#include <pthread.h>\nint main(void) { return 0; }' | $(CC) $(FLAGS) -o have_pthread$(EXT) -x c - -pthread 2> $(VOID) && rm have_pthread$(EXT) && echo 1 || echo 0)
+HAVE_PTHREAD := $(shell printf '\#include <pthread.h>\nint main(void) { return 0; }' > have_pthread.c && $(CC) $(FLAGS) -o have_pthread$(EXT) have_pthread.c -pthread 2> $(VOID) && rm have_pthread$(EXT) && echo 1 || echo 0; rm have_pthread.c)
HAVE_THREAD := $(shell [ "$(HAVE_PTHREAD)" -eq "1" -o -n "$(filter Windows%,$(OS))" ] && echo 1 || echo 0)
ifeq ($(HAVE_THREAD), 1)
THREAD_MSG := ==> building with threading support
@@ -103,7 +108,7 @@ endif
# zlib detection
NO_ZLIB_MSG := ==> no zlib, building zstd without .gz support
-HAVE_ZLIB := $(shell printf '\#include <zlib.h>\nint main(void) { return 0; }' | $(CC) $(FLAGS) -o have_zlib$(EXT) -x c - -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0)
+HAVE_ZLIB := $(shell printf '\#include <zlib.h>\nint main(void) { return 0; }' > have_zlib.c && $(CC) $(FLAGS) -o have_zlib$(EXT) have_zlib.c -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0; rm have_zlib.c)
ifeq ($(HAVE_ZLIB), 1)
ZLIB_MSG := ==> building zstd with .gz compression support
ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS
@@ -114,7 +119,7 @@ endif
# lzma detection
NO_LZMA_MSG := ==> no liblzma, building zstd without .xz/.lzma support
-HAVE_LZMA := $(shell printf '\#include <lzma.h>\nint main(void) { return 0; }' | $(CC) $(FLAGS) -o have_lzma$(EXT) -x c - -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0)
+HAVE_LZMA := $(shell printf '\#include <lzma.h>\nint main(void) { return 0; }' > have_lzma.c && $(CC) $(FLAGS) -o have_lzma$(EXT) have_lzma.c -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0; rm have_lzma.c)
ifeq ($(HAVE_LZMA), 1)
LZMA_MSG := ==> building zstd with .xz/.lzma compression support
LZMACPP = -DZSTD_LZMACOMPRESS -DZSTD_LZMADECOMPRESS
@@ -125,7 +130,7 @@ endif
# lz4 detection
NO_LZ4_MSG := ==> no liblz4, building zstd without .lz4 support
-HAVE_LZ4 := $(shell printf '\#include <lz4frame.h>\n\#include <lz4.h>\nint main(void) { return 0; }' | $(CC) $(FLAGS) -o have_lz4$(EXT) -x c - -llz4 2> $(VOID) && rm have_lz4$(EXT) && echo 1 || echo 0)
+HAVE_LZ4 := $(shell printf '\#include <lz4frame.h>\n\#include <lz4.h>\nint main(void) { return 0; }' > have_lz4.c && $(CC) $(FLAGS) -o have_lz4$(EXT) have_lz4.c -llz4 2> $(VOID) && rm have_lz4$(EXT) && echo 1 || echo 0; rm have_lz4.c)
ifeq ($(HAVE_LZ4), 1)
LZ4_MSG := ==> building zstd with .lz4 compression support
LZ4CPP = -DZSTD_LZ4COMPRESS -DZSTD_LZ4DECOMPRESS
@@ -160,7 +165,7 @@ $(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)
zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP)
zstd : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD)
zstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
-zstd : $(ZSTDLIB_FILES) zstdcli.o fileio.o bench.o datagen.o dibio.o
+zstd : $(ZSTDLIB_FILES) zstdcli.o util.o fileio.o benchfn.o benchzstd.o datagen.o dibio.o
@echo "$(THREAD_MSG)"
@echo "$(ZLIB_MSG)"
@echo "$(LZMA_MSG)"
@@ -178,13 +183,13 @@ zstd-release: zstd
zstd32 : CPPFLAGS += $(THREAD_CPP)
zstd32 : LDFLAGS += $(THREAD_LD)
zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
-zstd32 : $(ZSTDLIB_FILES) zstdcli.c fileio.c bench.c datagen.c dibio.c
+zstd32 : $(ZSTDLIB_FILES) zstdcli.c util.c fileio.c benchfn.c benchzstd.c datagen.c dibio.c
ifneq (,$(filter Windows%,$(OS)))
windres/generate_res.bat
endif
$(CC) -m32 $(FLAGS) $^ $(RES32_FILE) -o $@$(EXT)
-zstd-nolegacy : $(ZSTD_FILES) $(ZDICT_FILES) zstdcli.o fileio.c bench.o datagen.o dibio.o
+zstd-nolegacy : $(ZSTD_FILES) $(ZDICT_FILES) zstdcli.o util.o fileio.c benchfn.o benchzstd.o datagen.o dibio.o
$(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS)
zstd-nomt : THREAD_CPP :=
@@ -203,27 +208,27 @@ zstd-noxz : LZMA_MSG := - xz/lzma support is disabled
zstd-noxz : zstd
-zstd-pgo : MOREFLAGS = -fprofile-generate
-zstd-pgo : clean zstd
+zstd-pgo :
+ $(MAKE) clean
+ $(MAKE) zstd MOREFLAGS=-fprofile-generate
./zstd -b19i1 $(PROFILE_WITH)
./zstd -b16i1 $(PROFILE_WITH)
./zstd -b9i2 $(PROFILE_WITH)
./zstd -b $(PROFILE_WITH)
./zstd -b7i2 $(PROFILE_WITH)
./zstd -b5 $(PROFILE_WITH)
- $(RM) zstd
- $(RM) $(ZSTDDECOMP_O)
+ $(RM) zstd *.o $(ZSTDDECOMP_O) $(ZSTDDIR)/compress/*.o
$(MAKE) zstd MOREFLAGS=-fprofile-use
# minimal target, with only zstd compression and decompression. no bench. no legacy.
zstd-small: CFLAGS = -Os -s
-zstd-frugal zstd-small: $(ZSTD_FILES) zstdcli.c fileio.c
+zstd-frugal zstd-small: $(ZSTD_FILES) zstdcli.c util.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT $^ -o $@$(EXT)
-zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c fileio.c
+zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c util.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS $^ -o $@$(EXT)
-zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c fileio.c
+zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c util.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT)
zstdmt: zstd
@@ -275,7 +280,12 @@ preview-man: clean-man man
#-----------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku))
-EGREP = egrep --color=never
+HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
+EGREP_OPTIONS ?=
+ifeq ($HAVE_COLORNEVER, 1)
+EGREP_OPTIONS += --color=never
+endif
+EGREP = egrep $(EGREP_OPTIONS)
# Print a two column output of targets and their description. To add a target description, put a
# comment in the Makefile with the format "## <TARGET>: <DESCRIPTION>". For example:
@@ -352,6 +362,7 @@ uninstall:
@$(RM) $(DESTDIR)$(BINDIR)/zstdless
@$(RM) $(DESTDIR)$(BINDIR)/zstdcat
@$(RM) $(DESTDIR)$(BINDIR)/unzstd
+ @$(RM) $(DESTDIR)$(BINDIR)/zstdmt
@$(RM) $(DESTDIR)$(BINDIR)/zstd
@$(RM) $(DESTDIR)$(MAN1DIR)/zstdless.1
@$(RM) $(DESTDIR)$(MAN1DIR)/zstdgrep.1
diff --git a/programs/README.md b/programs/README.md
index ca9056eaaa437..afbebaa16fdf3 100644
--- a/programs/README.md
+++ b/programs/README.md
@@ -172,6 +172,11 @@ Benchmark arguments :
--priority=rt : set process priority to real-time
```
+#### Restricted usage of Environment Variables
+Using environment variables to set compression/decompression parameters has security implications. Therefore,
+we intentionally restrict its usage. Currently, only `ZSTD_CLEVEL` is supported for setting compression level.
+If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a warning message.
+Note that command line options will override corresponding environment variable settings.
#### Long distance matching mode
The long distance matching mode, enabled with `--long`, is designed to improve
diff --git a/programs/benchfn.c b/programs/benchfn.c
new file mode 100644
index 0000000000000..f5118d0871f11
--- /dev/null
+++ b/programs/benchfn.c
@@ -0,0 +1,263 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+
+/* *************************************
+* Includes
+***************************************/
+#include "platform.h" /* Large Files support */
+#include "util.h" /* UTIL_getFileSize, UTIL_sleep */
+#include <stdlib.h> /* malloc, free */
+#include <string.h> /* memset */
+#include <stdio.h> /* fprintf, fopen */
+#undef NDEBUG /* assert must not be disabled */
+#include <assert.h> /* assert */
+
+#include "mem.h"
+#include "benchfn.h"
+
+
+/* *************************************
+* Constants
+***************************************/
+#define TIMELOOP_MICROSEC (1*1000000ULL) /* 1 second */
+#define TIMELOOP_NANOSEC (1*1000000000ULL) /* 1 second */
+#define ACTIVEPERIOD_MICROSEC (70*TIMELOOP_MICROSEC) /* 70 seconds */
+#define COOLPERIOD_SEC 10
+
+#define KB *(1 <<10)
+#define MB *(1 <<20)
+#define GB *(1U<<30)
+
+
+/* *************************************
+* Errors
+***************************************/
+#ifndef DEBUG
+# define DEBUG 0
+#endif
+
+#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
+#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); }
+
+/* error without displaying */
+#define RETURN_QUIET_ERROR(retValue, ...) { \
+ DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
+ DEBUGOUTPUT("Error : "); \
+ DEBUGOUTPUT(__VA_ARGS__); \
+ DEBUGOUTPUT(" \n"); \
+ return retValue; \
+}
+
+
+/* *************************************
+* Benchmarking an arbitrary function
+***************************************/
+
+int BMK_isSuccessful_runOutcome(BMK_runOutcome_t outcome)
+{
+ return outcome.error_tag_never_ever_use_directly == 0;
+}
+
+/* warning : this function will stop program execution if outcome is invalid !
+ * check outcome validity first, using BMK_isValid_runResult() */
+BMK_runTime_t BMK_extract_runTime(BMK_runOutcome_t outcome)
+{
+ assert(outcome.error_tag_never_ever_use_directly == 0);
+ return outcome.internal_never_ever_use_directly;
+}
+
+size_t BMK_extract_errorResult(BMK_runOutcome_t outcome)
+{
+ assert(outcome.error_tag_never_ever_use_directly != 0);
+ return outcome.error_result_never_ever_use_directly;
+}
+
+static BMK_runOutcome_t BMK_runOutcome_error(size_t errorResult)
+{
+ BMK_runOutcome_t b;
+ memset(&b, 0, sizeof(b));
+ b.error_tag_never_ever_use_directly = 1;
+ b.error_result_never_ever_use_directly = errorResult;
+ return b;
+}
+
+static BMK_runOutcome_t BMK_setValid_runTime(BMK_runTime_t runTime)
+{
+ BMK_runOutcome_t outcome;
+ outcome.error_tag_never_ever_use_directly = 0;
+ outcome.internal_never_ever_use_directly = runTime;
+ return outcome;
+}
+
+
+/* initFn will be measured once, benchFn will be measured `nbLoops` times */
+/* initFn is optional, provide NULL if none */
+/* benchFn must return a size_t value that errorFn can interpret */
+/* takes # of blocks and list of size & stuff for each. */
+/* can report result of benchFn for each block into blockResult. */
+/* blockResult is optional, provide NULL if this information is not required */
+/* note : time per loop can be reported as zero if run time < timer resolution */
+BMK_runOutcome_t BMK_benchFunction(BMK_benchParams_t p,
+ unsigned nbLoops)
+{
+ size_t dstSize = 0;
+ nbLoops += !nbLoops; /* minimum nbLoops is 1 */
+
+ /* init */
+ { size_t i;
+ for(i = 0; i < p.blockCount; i++) {
+ memset(p.dstBuffers[i], 0xE5, p.dstCapacities[i]); /* warm up and erase result buffer */
+ }
+#if 0
+ /* based on testing these seem to lower accuracy of multiple calls of 1 nbLoops vs 1 call of multiple nbLoops
+ * (Makes former slower)
+ */
+ UTIL_sleepMilli(5); /* give processor time to other processes */
+ UTIL_waitForNextTick();
+#endif
+ }
+
+ /* benchmark */
+ { UTIL_time_t const clockStart = UTIL_getTime();
+ unsigned loopNb, blockNb;
+ if (p.initFn != NULL) p.initFn(p.initPayload);
+ for (loopNb = 0; loopNb < nbLoops; loopNb++) {
+ for (blockNb = 0; blockNb < p.blockCount; blockNb++) {
+ size_t const res = p.benchFn(p.srcBuffers[blockNb], p.srcSizes[blockNb],
+ p.dstBuffers[blockNb], p.dstCapacities[blockNb],
+ p.benchPayload);
+ if (loopNb == 0) {
+ if (p.blockResults != NULL) p.blockResults[blockNb] = res;
+ if ((p.errorFn != NULL) && (p.errorFn(res))) {
+ RETURN_QUIET_ERROR(BMK_runOutcome_error(res),
+ "Function benchmark failed on block %u (of size %u) with error %i",
+ blockNb, (unsigned)p.srcSizes[blockNb], (int)res);
+ }
+ dstSize += res;
+ } }
+ } /* for (loopNb = 0; loopNb < nbLoops; loopNb++) */
+
+ { U64 const totalTime = UTIL_clockSpanNano(clockStart);
+ BMK_runTime_t rt;
+ rt.nanoSecPerRun = totalTime / nbLoops;
+ rt.sumOfReturn = dstSize;
+ return BMK_setValid_runTime(rt);
+ } }
+}
+
+
+/* ==== Benchmarking any function, providing intermediate results ==== */
+
+struct BMK_timedFnState_s {
+ U64 timeSpent_ns;
+ U64 timeBudget_ns;
+ U64 runBudget_ns;
+ BMK_runTime_t fastestRun;
+ unsigned nbLoops;
+ UTIL_time_t coolTime;
+}; /* typedef'd to BMK_timedFnState_t within bench.h */
+
+BMK_timedFnState_t* BMK_createTimedFnState(unsigned total_ms, unsigned run_ms)
+{
+ BMK_timedFnState_t* const r = (BMK_timedFnState_t*)malloc(sizeof(*r));
+ if (r == NULL) return NULL; /* malloc() error */
+ BMK_resetTimedFnState(r, total_ms, run_ms);
+ return r;
+}
+
+void BMK_freeTimedFnState(BMK_timedFnState_t* state) {
+ free(state);
+}
+
+void BMK_resetTimedFnState(BMK_timedFnState_t* timedFnState, unsigned total_ms, unsigned run_ms)
+{
+ if (!total_ms) total_ms = 1 ;
+ if (!run_ms) run_ms = 1;
+ if (run_ms > total_ms) run_ms = total_ms;
+ timedFnState->timeSpent_ns = 0;
+ timedFnState->timeBudget_ns = (U64)total_ms * TIMELOOP_NANOSEC / 1000;
+ timedFnState->runBudget_ns = (U64)run_ms * TIMELOOP_NANOSEC / 1000;
+ timedFnState->fastestRun.nanoSecPerRun = (U64)(-1LL);
+ timedFnState->fastestRun.sumOfReturn = (size_t)(-1LL);
+ timedFnState->nbLoops = 1;
+ timedFnState->coolTime = UTIL_getTime();
+}
+
+/* Tells if nb of seconds set in timedFnState for all runs is spent.
+ * note : this function will return 1 if BMK_benchFunctionTimed() has actually errored. */
+int BMK_isCompleted_TimedFn(const BMK_timedFnState_t* timedFnState)
+{
+ return (timedFnState->timeSpent_ns >= timedFnState->timeBudget_ns);
+}
+
+
+#undef MIN
+#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
+
+#define MINUSABLETIME (TIMELOOP_NANOSEC / 2) /* 0.5 seconds */
+
+BMK_runOutcome_t BMK_benchTimedFn(BMK_timedFnState_t* cont,
+ BMK_benchParams_t p)
+{
+ U64 const runBudget_ns = cont->runBudget_ns;
+ U64 const runTimeMin_ns = runBudget_ns / 2;
+ int completed = 0;
+ BMK_runTime_t bestRunTime = cont->fastestRun;
+
+ while (!completed) {
+ BMK_runOutcome_t runResult;
+
+ /* Overheat protection */
+ if (UTIL_clockSpanMicro(cont->coolTime) > ACTIVEPERIOD_MICROSEC) {
+ DEBUGOUTPUT("\rcooling down ... \r");
+ UTIL_sleep(COOLPERIOD_SEC);
+ cont->coolTime = UTIL_getTime();
+ }
+
+ /* reinitialize capacity */
+ runResult = BMK_benchFunction(p, cont->nbLoops);
+
+ if(!BMK_isSuccessful_runOutcome(runResult)) { /* error : move out */
+ return runResult;
+ }
+
+ { BMK_runTime_t const newRunTime = BMK_extract_runTime(runResult);
+ U64 const loopDuration_ns = newRunTime.nanoSecPerRun * cont->nbLoops;
+
+ cont->timeSpent_ns += loopDuration_ns;
+
+ /* estimate nbLoops for next run to last approximately 1 second */
+ if (loopDuration_ns > (runBudget_ns / 50)) {
+ U64 const fastestRun_ns = MIN(bestRunTime.nanoSecPerRun, newRunTime.nanoSecPerRun);
+ cont->nbLoops = (U32)(runBudget_ns / fastestRun_ns) + 1;
+ } else {
+ /* previous run was too short : blindly increase workload by x multiplier */
+ const unsigned multiplier = 10;
+ assert(cont->nbLoops < ((unsigned)-1) / multiplier); /* avoid overflow */
+ cont->nbLoops *= multiplier;
+ }
+
+ if(loopDuration_ns < runTimeMin_ns) {
+ /* don't report results for which benchmark run time was too small : increased risks of rounding errors */
+ assert(completed == 0);
+ continue;
+ } else {
+ if(newRunTime.nanoSecPerRun < bestRunTime.nanoSecPerRun) {
+ bestRunTime = newRunTime;
+ }
+ completed = 1;
+ }
+ }
+ } /* while (!completed) */
+
+ return BMK_setValid_runTime(bestRunTime);
+}
diff --git a/programs/benchfn.h b/programs/benchfn.h
new file mode 100644
index 0000000000000..3ca36e3623dab
--- /dev/null
+++ b/programs/benchfn.h
@@ -0,0 +1,167 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+/* benchfn :
+ * benchmark any function on a set of input
+ * providing result in nanoSecPerRun
+ * or detecting and returning an error
+ */
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#ifndef BENCH_FN_H_23876
+#define BENCH_FN_H_23876
+
+/* === Dependencies === */
+#include <stddef.h> /* size_t */
+
+
+/* ==== Benchmark any function, iterated on a set of blocks ==== */
+
+/* BMK_runTime_t: valid result return type */
+
+typedef struct {
+ unsigned long long nanoSecPerRun; /* time per iteration (over all blocks) */
+ size_t sumOfReturn; /* sum of return values */
+} BMK_runTime_t;
+
+
+/* BMK_runOutcome_t:
+ * type expressing the outcome of a benchmark run by BMK_benchFunction(),
+ * which can be either valid or invalid.
+ * benchmark outcome can be invalid if errorFn is provided.
+ * BMK_runOutcome_t must be considered "opaque" : never access its members directly.
+ * Instead, use its assigned methods :
+ * BMK_isSuccessful_runOutcome, BMK_extract_runTime, BMK_extract_errorResult.
+ * The structure is only described here to allow its allocation on stack. */
+
+typedef struct {
+ BMK_runTime_t internal_never_ever_use_directly;
+ size_t error_result_never_ever_use_directly;
+ int error_tag_never_ever_use_directly;
+} BMK_runOutcome_t;
+
+
+/* prototypes for benchmarked functions */
+typedef size_t (*BMK_benchFn_t)(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* customPayload);
+typedef size_t (*BMK_initFn_t)(void* initPayload);
+typedef unsigned (*BMK_errorFn_t)(size_t);
+
+
+/* BMK_benchFunction() parameters are provided through following structure.
+ * This is preferable for readability,
+ * as the number of parameters required is pretty large.
+ * No initializer is provided, because it doesn't make sense to provide some "default" :
+ * all parameters should be specified by the caller */
+typedef struct {
+ BMK_benchFn_t benchFn; /* the function to benchmark, over the set of blocks */
+ void* benchPayload; /* pass custom parameters to benchFn :
+ * (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstCapacities[i], benchPayload) */
+ BMK_initFn_t initFn; /* (*initFn)(initPayload) is run once per run, at the beginning. */
+ void* initPayload; /* Both arguments can be NULL, in which case nothing is run. */
+ BMK_errorFn_t errorFn; /* errorFn will check each return value of benchFn over each block, to determine if it failed or not.
+ * errorFn can be NULL, in which case no check is performed.
+ * errorFn must return 0 when benchFn was successful, and >= 1 if it detects an error.
+ * Execution is stopped as soon as an error is detected.
+ * the triggering return value can be retrieved using BMK_extract_errorResult(). */
+ size_t blockCount; /* number of blocks to operate benchFn on.
+ * It's also the size of all array parameters :
+ * srcBuffers, srcSizes, dstBuffers, dstCapacities, blockResults */
+ const void *const * srcBuffers; /* array of buffers to be operated on by benchFn */
+ const size_t* srcSizes; /* array of the sizes of srcBuffers buffers */
+ void *const * dstBuffers;/* array of buffers to be written into by benchFn */
+ const size_t* dstCapacities; /* array of the capacities of dstBuffers buffers */
+ size_t* blockResults; /* Optional: store the return value of benchFn for each block. Use NULL if this result is not requested. */
+} BMK_benchParams_t;
+
+
+/* BMK_benchFunction() :
+ * This function benchmarks benchFn and initFn, providing a result.
+ *
+ * params : see description of BMK_benchParams_t above.
+ * nbLoops: defines number of times benchFn is run over the full set of blocks.
+ * Minimum value is 1. A 0 is interpreted as a 1.
+ *
+ * @return: can express either an error or a successful result.
+ * Use BMK_isSuccessful_runOutcome() to check if benchmark was successful.
+ * If yes, extract the result with BMK_extract_runTime(),
+ * it will contain :
+ * .sumOfReturn : the sum of all return values of benchFn through all of blocks
+ * .nanoSecPerRun : time per run of benchFn + (time for initFn / nbLoops)
+ * .sumOfReturn is generally intended for functions which return a # of bytes written into dstBuffer,
+ * in which case, this value will be the total amount of bytes written into dstBuffer.
+ *
+ * blockResults : when provided (!= NULL), and when benchmark is successful,
+ * params.blockResults contains all return values of `benchFn` over all blocks.
+ * when provided (!= NULL), and when benchmark failed,
+ * params.blockResults contains return values of `benchFn` over all blocks preceding and including the failed block.
+ */
+BMK_runOutcome_t BMK_benchFunction(BMK_benchParams_t params, unsigned nbLoops);
+
+
+
+/* check first if the benchmark was successful or not */
+int BMK_isSuccessful_runOutcome(BMK_runOutcome_t outcome);
+
+/* If the benchmark was successful, extract the result.
+ * note : this function will abort() program execution if benchmark failed !
+ * always check if benchmark was successful first !
+ */
+BMK_runTime_t BMK_extract_runTime(BMK_runOutcome_t outcome);
+
+/* when benchmark failed, it means one invocation of `benchFn` failed.
+ * The failure was detected by `errorFn`, operating on return values of `benchFn`.
+ * Returns the faulty return value.
+ * note : this function will abort() program execution if benchmark did not failed.
+ * always check if benchmark failed first !
+ */
+size_t BMK_extract_errorResult(BMK_runOutcome_t outcome);
+
+
+
+/* ==== Benchmark any function, returning intermediate results ==== */
+
+/* state information tracking benchmark session */
+typedef struct BMK_timedFnState_s BMK_timedFnState_t;
+
+/* BMK_benchTimedFn() :
+ * Similar to BMK_benchFunction(), most arguments being identical.
+ * Automatically determines `nbLoops` so that each result is regularly produced at interval of about run_ms.
+ * Note : minimum `nbLoops` is 1, therefore a run may last more than run_ms, and possibly even more than total_ms.
+ * Usage - initialize timedFnState, select benchmark duration (total_ms) and each measurement duration (run_ms)
+ * call BMK_benchTimedFn() repetitively, each measurement is supposed to last about run_ms
+ * Check if total time budget is spent or exceeded, using BMK_isCompleted_TimedFn()
+ */
+BMK_runOutcome_t BMK_benchTimedFn(BMK_timedFnState_t* timedFnState,
+ BMK_benchParams_t params);
+
+/* Tells if duration of all benchmark runs has exceeded total_ms
+ */
+int BMK_isCompleted_TimedFn(const BMK_timedFnState_t* timedFnState);
+
+/* BMK_createTimedFnState() and BMK_resetTimedFnState() :
+ * Create/Set BMK_timedFnState_t for next benchmark session,
+ * which shall last a minimum of total_ms milliseconds,
+ * producing intermediate results, paced at interval of (approximately) run_ms.
+ */
+BMK_timedFnState_t* BMK_createTimedFnState(unsigned total_ms, unsigned run_ms);
+void BMK_resetTimedFnState(BMK_timedFnState_t* timedFnState, unsigned total_ms, unsigned run_ms);
+void BMK_freeTimedFnState(BMK_timedFnState_t* state);
+
+
+
+#endif /* BENCH_FN_H_23876 */
+
+#if defined (__cplusplus)
+}
+#endif
diff --git a/programs/bench.c b/programs/benchzstd.c
index 326c1c1c56e59..0be74a965047e 100644
--- a/programs/bench.c
+++ b/programs/benchzstd.c
@@ -9,7 +9,6 @@
*/
-
/* **************************************
* Tuning parameters
****************************************/
@@ -18,30 +17,24 @@
#endif
-/* **************************************
-* Compiler Warnings
-****************************************/
-#ifdef _MSC_VER
-# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
-#endif
-
-
/* *************************************
* Includes
***************************************/
#include "platform.h" /* Large Files support */
#include "util.h" /* UTIL_getFileSize, UTIL_sleep */
#include <stdlib.h> /* malloc, free */
-#include <string.h> /* memset */
+#include <string.h> /* memset, strerror */
#include <stdio.h> /* fprintf, fopen */
+#include <errno.h>
#include <assert.h> /* assert */
+#include "benchfn.h"
#include "mem.h"
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"
#include "datagen.h" /* RDG_genBuffer */
#include "xxhash.h"
-#include "bench.h"
+#include "benchzstd.h"
#include "zstd_errors.h"
@@ -102,6 +95,18 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
return errorNum; \
}
+#define CHECK_Z(zf) { \
+ size_t const zerr = zf; \
+ if (ZSTD_isError(zerr)) { \
+ DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
+ DISPLAY("Error : "); \
+ DISPLAY("%s failed : %s", \
+ #zf, ZSTD_getErrorName(zerr)); \
+ DISPLAY(" \n"); \
+ exit(1); \
+ } \
+}
+
#define RETURN_ERROR(errorNum, retType, ...) { \
retType r; \
memset(&r, 0, sizeof(retType)); \
@@ -113,17 +118,6 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
return r; \
}
-/* error without displaying */
-#define RETURN_QUIET_ERROR(errorNum, retType, ...) { \
- retType r; \
- memset(&r, 0, sizeof(retType)); \
- DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
- DEBUGOUTPUT("Error %i : ", errorNum); \
- DEBUGOUTPUT(__VA_ARGS__); \
- DEBUGOUTPUT(" \n"); \
- r.tag = errorNum; \
- return r; \
-}
/* *************************************
* Benchmark Parameters
@@ -141,7 +135,7 @@ BMK_advancedParams_t BMK_initAdvancedParams(void) {
0, /* ldmMinMatch */
0, /* ldmHashLog */
0, /* ldmBuckSizeLog */
- 0 /* ldmHashEveryLog */
+ 0 /* ldmHashRateLog */
};
return res;
}
@@ -168,33 +162,32 @@ typedef struct {
static void BMK_initCCtx(ZSTD_CCtx* ctx,
const void* dictBuffer, size_t dictBufferSize, int cLevel,
const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) {
- ZSTD_CCtx_reset(ctx);
- ZSTD_CCtx_resetParameters(ctx);
+ ZSTD_CCtx_reset(ctx, ZSTD_reset_session_and_parameters);
if (adv->nbWorkers==1) {
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0);
+ CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_nbWorkers, 0));
} else {
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, adv->nbWorkers);
+ CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_nbWorkers, adv->nbWorkers));
}
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel);
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, adv->ldmFlag);
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmMinMatch, adv->ldmMinMatch);
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashLog, adv->ldmHashLog);
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmBucketSizeLog, adv->ldmBucketSizeLog);
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashEveryLog, adv->ldmHashEveryLog);
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_windowLog, comprParams->windowLog);
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_hashLog, comprParams->hashLog);
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_chainLog, comprParams->chainLog);
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog);
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength);
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength);
- ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy);
- ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
+ CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, cLevel));
+ CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_enableLongDistanceMatching, adv->ldmFlag));
+ CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmMinMatch, adv->ldmMinMatch));
+ CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashLog, adv->ldmHashLog));
+ CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmBucketSizeLog, adv->ldmBucketSizeLog));
+ CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashRateLog, adv->ldmHashRateLog));
+ CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_windowLog, comprParams->windowLog));
+ CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_hashLog, comprParams->hashLog));
+ CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_chainLog, comprParams->chainLog));
+ CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_searchLog, comprParams->searchLog));
+ CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_minMatch, comprParams->minMatch));
+ CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_targetLength, comprParams->targetLength));
+ CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_strategy, comprParams->strategy));
+ CHECK_Z(ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize));
}
static void BMK_initDCtx(ZSTD_DCtx* dctx,
const void* dictBuffer, size_t dictBufferSize) {
- ZSTD_DCtx_reset(dctx);
- ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize);
+ CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters));
+ CHECK_Z(ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize));
}
@@ -232,22 +225,8 @@ static size_t local_defaultCompress(
void* dstBuffer, size_t dstSize,
void* addArgs)
{
- size_t moreToFlush = 1;
ZSTD_CCtx* const cctx = (ZSTD_CCtx*)addArgs;
- ZSTD_inBuffer in;
- ZSTD_outBuffer out;
- in.src = srcBuffer; in.size = srcSize; in.pos = 0;
- out.dst = dstBuffer; out.size = dstSize; out.pos = 0;
- while (moreToFlush) {
- if(out.pos == out.size) {
- return (size_t)-ZSTD_error_dstSize_tooSmall;
- }
- moreToFlush = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
- if (ZSTD_isError(moreToFlush)) {
- return moreToFlush;
- }
- }
- return out.pos;
+ return ZSTD_compress2(cctx, dstBuffer, dstSize, srcBuffer, srcSize);
}
/* `addArgs` is the context */
@@ -266,7 +245,7 @@ static size_t local_defaultDecompress(
if(out.pos == out.size) {
return (size_t)-ZSTD_error_dstSize_tooSmall;
}
- moreToFlush = ZSTD_decompress_generic(dctx, &out, &in);
+ moreToFlush = ZSTD_decompressStream(dctx, &out, &in);
if (ZSTD_isError(moreToFlush)) {
return moreToFlush;
}
@@ -276,219 +255,6 @@ static size_t local_defaultDecompress(
}
-/*=== Benchmarking an arbitrary function ===*/
-
-int BMK_isSuccessful_runOutcome(BMK_runOutcome_t outcome)
-{
- return outcome.tag == 0;
-}
-
-/* warning : this function will stop program execution if outcome is invalid !
- * check outcome validity first, using BMK_isValid_runResult() */
-BMK_runTime_t BMK_extract_runTime(BMK_runOutcome_t outcome)
-{
- assert(outcome.tag == 0);
- return outcome.internal_never_use_directly;
-}
-
-static BMK_runOutcome_t BMK_runOutcome_error(void)
-{
- BMK_runOutcome_t b;
- memset(&b, 0, sizeof(b));
- b.tag = 1;
- return b;
-}
-
-static BMK_runOutcome_t BMK_setValid_runTime(BMK_runTime_t runTime)
-{
- BMK_runOutcome_t outcome;
- outcome.tag = 0;
- outcome.internal_never_use_directly = runTime;
- return outcome;
-}
-
-
-/* initFn will be measured once, benchFn will be measured `nbLoops` times */
-/* initFn is optional, provide NULL if none */
-/* benchFn must return size_t field compliant with ZSTD_isError for error valuee */
-/* takes # of blocks and list of size & stuff for each. */
-/* can report result of benchFn for each block into blockResult. */
-/* blockResult is optional, provide NULL if this information is not required */
-/* note : time per loop could be zero if run time < timer resolution */
-BMK_runOutcome_t BMK_benchFunction(
- BMK_benchFn_t benchFn, void* benchPayload,
- BMK_initFn_t initFn, void* initPayload,
- size_t blockCount,
- const void* const * srcBlockBuffers, const size_t* srcBlockSizes,
- void* const * dstBlockBuffers, const size_t* dstBlockCapacities,
- size_t* blockResults,
- unsigned nbLoops)
-{
- size_t dstSize = 0;
-
- if(!nbLoops) {
- RETURN_QUIET_ERROR(2, BMK_runOutcome_t, "nbLoops must be nonzero ");
- }
-
- /* init */
- { size_t i;
- for(i = 0; i < blockCount; i++) {
- memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */
- }
-#if 0
- /* based on testing these seem to lower accuracy of multiple calls of 1 nbLoops vs 1 call of multiple nbLoops
- * (Makes former slower)
- */
- UTIL_sleepMilli(5); /* give processor time to other processes */
- UTIL_waitForNextTick();
-#endif
- }
-
- /* benchmark */
- { UTIL_time_t const clockStart = UTIL_getTime();
- unsigned loopNb, blockNb;
- if (initFn != NULL) initFn(initPayload);
- for (loopNb = 0; loopNb < nbLoops; loopNb++) {
- for (blockNb = 0; blockNb < blockCount; blockNb++) {
- size_t const res = benchFn(srcBlockBuffers[blockNb], srcBlockSizes[blockNb],
- dstBlockBuffers[blockNb], dstBlockCapacities[blockNb],
- benchPayload);
- if(ZSTD_isError(res)) {
- RETURN_QUIET_ERROR(2, BMK_runOutcome_t,
- "Function benchmark failed on block %u of size %u : %s",
- blockNb, (U32)dstBlockCapacities[blockNb], ZSTD_getErrorName(res));
- } else if (loopNb == 0) {
- dstSize += res;
- if (blockResults != NULL) blockResults[blockNb] = res;
- } }
- } /* for (loopNb = 0; loopNb < nbLoops; loopNb++) */
-
- { U64 const totalTime = UTIL_clockSpanNano(clockStart);
- BMK_runTime_t rt;
- rt.nanoSecPerRun = totalTime / nbLoops;
- rt.sumOfReturn = dstSize;
- return BMK_setValid_runTime(rt);
- } }
-}
-
-
-/* ==== Benchmarking any function, providing intermediate results ==== */
-
-struct BMK_timedFnState_s {
- U64 timeSpent_ns;
- U64 timeBudget_ns;
- U64 runBudget_ns;
- BMK_runTime_t fastestRun;
- unsigned nbLoops;
- UTIL_time_t coolTime;
-}; /* typedef'd to BMK_timedFnState_t within bench.h */
-
-BMK_timedFnState_t* BMK_createTimedFnState(unsigned total_ms, unsigned run_ms)
-{
- BMK_timedFnState_t* const r = (BMK_timedFnState_t*)malloc(sizeof(*r));
- if (r == NULL) return NULL; /* malloc() error */
- BMK_resetTimedFnState(r, total_ms, run_ms);
- return r;
-}
-
-void BMK_freeTimedFnState(BMK_timedFnState_t* state) {
- free(state);
-}
-
-void BMK_resetTimedFnState(BMK_timedFnState_t* timedFnState, unsigned total_ms, unsigned run_ms)
-{
- if (!total_ms) total_ms = 1 ;
- if (!run_ms) run_ms = 1;
- if (run_ms > total_ms) run_ms = total_ms;
- timedFnState->timeSpent_ns = 0;
- timedFnState->timeBudget_ns = (U64)total_ms * TIMELOOP_NANOSEC / 1000;
- timedFnState->runBudget_ns = (U64)run_ms * TIMELOOP_NANOSEC / 1000;
- timedFnState->fastestRun.nanoSecPerRun = (U64)(-1LL);
- timedFnState->fastestRun.sumOfReturn = (size_t)(-1LL);
- timedFnState->nbLoops = 1;
- timedFnState->coolTime = UTIL_getTime();
-}
-
-/* Tells if nb of seconds set in timedFnState for all runs is spent.
- * note : this function will return 1 if BMK_benchFunctionTimed() has actually errored. */
-int BMK_isCompleted_TimedFn(const BMK_timedFnState_t* timedFnState)
-{
- return (timedFnState->timeSpent_ns >= timedFnState->timeBudget_ns);
-}
-
-
-#define MINUSABLETIME (TIMELOOP_NANOSEC / 2) /* 0.5 seconds */
-
-BMK_runOutcome_t BMK_benchTimedFn(
- BMK_timedFnState_t* cont,
- BMK_benchFn_t benchFn, void* benchPayload,
- BMK_initFn_t initFn, void* initPayload,
- size_t blockCount,
- const void* const* srcBlockBuffers, const size_t* srcBlockSizes,
- void * const * dstBlockBuffers, const size_t * dstBlockCapacities,
- size_t* blockResults)
-{
- U64 const runBudget_ns = cont->runBudget_ns;
- U64 const runTimeMin_ns = runBudget_ns / 2;
- int completed = 0;
- BMK_runTime_t bestRunTime = cont->fastestRun;
-
- while (!completed) {
- BMK_runOutcome_t runResult;
-
- /* Overheat protection */
- if (UTIL_clockSpanMicro(cont->coolTime) > ACTIVEPERIOD_MICROSEC) {
- DEBUGOUTPUT("\rcooling down ... \r");
- UTIL_sleep(COOLPERIOD_SEC);
- cont->coolTime = UTIL_getTime();
- }
-
- /* reinitialize capacity */
- runResult = BMK_benchFunction(benchFn, benchPayload,
- initFn, initPayload,
- blockCount,
- srcBlockBuffers, srcBlockSizes,
- dstBlockBuffers, dstBlockCapacities,
- blockResults,
- cont->nbLoops);
-
- if(!BMK_isSuccessful_runOutcome(runResult)) { /* error : move out */
- return BMK_runOutcome_error();
- }
-
- { BMK_runTime_t const newRunTime = BMK_extract_runTime(runResult);
- U64 const loopDuration_ns = newRunTime.nanoSecPerRun * cont->nbLoops;
-
- cont->timeSpent_ns += loopDuration_ns;
-
- /* estimate nbLoops for next run to last approximately 1 second */
- if (loopDuration_ns > (runBudget_ns / 50)) {
- U64 const fastestRun_ns = MIN(bestRunTime.nanoSecPerRun, newRunTime.nanoSecPerRun);
- cont->nbLoops = (U32)(runBudget_ns / fastestRun_ns) + 1;
- } else {
- /* previous run was too short : blindly increase workload by x multiplier */
- const unsigned multiplier = 10;
- assert(cont->nbLoops < ((unsigned)-1) / multiplier); /* avoid overflow */
- cont->nbLoops *= multiplier;
- }
-
- if(loopDuration_ns < runTimeMin_ns) {
- /* don't report results for which benchmark run time was too small : increased risks of rounding errors */
- assert(completed == 0);
- continue;
- } else {
- if(newRunTime.nanoSecPerRun < bestRunTime.nanoSecPerRun) {
- bestRunTime = newRunTime;
- }
- completed = 1;
- }
- }
- } /* while (!completed) */
-
- return BMK_setValid_runTime(bestRunTime);
-}
-
-
/* ================================================================= */
/* Benchmark Zstandard, mem-to-mem scenarios */
/* ================================================================= */
@@ -522,22 +288,24 @@ static BMK_benchOutcome_t BMK_benchOutcome_setValidResult(BMK_benchResult_t resu
/* benchMem with no allocation */
-static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
- const void** srcPtrs, size_t* srcSizes,
- void** cPtrs, size_t* cCapacities, size_t* cSizes,
- void** resPtrs, size_t* resSizes,
- void** resultBufferPtr, void* compressedBuffer,
- size_t maxCompressedSize,
- BMK_timedFnState_t* timeStateCompress,
- BMK_timedFnState_t* timeStateDecompress,
+static BMK_benchOutcome_t
+BMK_benchMemAdvancedNoAlloc(
+ const void** srcPtrs, size_t* srcSizes,
+ void** cPtrs, size_t* cCapacities, size_t* cSizes,
+ void** resPtrs, size_t* resSizes,
+ void** resultBufferPtr, void* compressedBuffer,
+ size_t maxCompressedSize,
+ BMK_timedFnState_t* timeStateCompress,
+ BMK_timedFnState_t* timeStateDecompress,
- const void* srcBuffer, size_t srcSize,
- const size_t* fileSizes, unsigned nbFiles,
- const int cLevel, const ZSTD_compressionParameters* comprParams,
- const void* dictBuffer, size_t dictBufferSize,
- ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
- int displayLevel, const char* displayName,
- const BMK_advancedParams_t* adv)
+ const void* srcBuffer, size_t srcSize,
+ const size_t* fileSizes, unsigned nbFiles,
+ const int cLevel,
+ const ZSTD_compressionParameters* comprParams,
+ const void* dictBuffer, size_t dictBufferSize,
+ ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
+ int displayLevel, const char* displayName,
+ const BMK_advancedParams_t* adv)
{
size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize); /* avoid div by 0 */
BMK_benchResult_t benchResult;
@@ -599,6 +367,11 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
cPtr += cCapacities[nbBlocks];
resPtr += thisBlockSize;
remaining -= thisBlockSize;
+ if (adv->mode == BMK_decodeOnly) {
+ assert(nbBlocks==0);
+ cSizes[nbBlocks] = thisBlockSize;
+ benchResult.cSize = thisBlockSize;
+ }
}
}
}
@@ -617,32 +390,51 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
U32 markNb = 0;
int compressionCompleted = (adv->mode == BMK_decodeOnly);
int decompressionCompleted = (adv->mode == BMK_compressOnly);
+ BMK_benchParams_t cbp, dbp;
BMK_initCCtxArgs cctxprep;
BMK_initDCtxArgs dctxprep;
+
+ cbp.benchFn = local_defaultCompress;
+ cbp.benchPayload = cctx;
+ cbp.initFn = local_initCCtx;
+ cbp.initPayload = &cctxprep;
+ cbp.errorFn = ZSTD_isError;
+ cbp.blockCount = nbBlocks;
+ cbp.srcBuffers = srcPtrs;
+ cbp.srcSizes = srcSizes;
+ cbp.dstBuffers = cPtrs;
+ cbp.dstCapacities = cCapacities;
+ cbp.blockResults = cSizes;
+
cctxprep.cctx = cctx;
cctxprep.dictBuffer = dictBuffer;
cctxprep.dictBufferSize = dictBufferSize;
cctxprep.cLevel = cLevel;
cctxprep.comprParams = comprParams;
cctxprep.adv = adv;
+
+ dbp.benchFn = local_defaultDecompress;
+ dbp.benchPayload = dctx;
+ dbp.initFn = local_initDCtx;
+ dbp.initPayload = &dctxprep;
+ dbp.errorFn = ZSTD_isError;
+ dbp.blockCount = nbBlocks;
+ dbp.srcBuffers = (const void* const *) cPtrs;
+ dbp.srcSizes = cSizes;
+ dbp.dstBuffers = resPtrs;
+ dbp.dstCapacities = resSizes;
+ dbp.blockResults = NULL;
+
dctxprep.dctx = dctx;
dctxprep.dictBuffer = dictBuffer;
dctxprep.dictBufferSize = dictBufferSize;
DISPLAYLEVEL(2, "\r%70s\r", ""); /* blank line */
- DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize);
+ DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (unsigned)srcSize);
while (!(compressionCompleted && decompressionCompleted)) {
-
if (!compressionCompleted) {
- BMK_runOutcome_t const cOutcome =
- BMK_benchTimedFn( timeStateCompress,
- &local_defaultCompress, cctx,
- &local_initCCtx, &cctxprep,
- nbBlocks,
- srcPtrs, srcSizes,
- cPtrs, cCapacities,
- cSizes);
+ BMK_runOutcome_t const cOutcome = BMK_benchTimedFn( timeStateCompress, cbp);
if (!BMK_isSuccessful_runOutcome(cOutcome)) {
return BMK_benchOutcome_error();
@@ -659,10 +451,9 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
} }
{ int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
- markNb = (markNb+1) % NB_MARKS;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
marks[markNb], displayName,
- (U32)srcSize, (U32)cSize,
+ (unsigned)srcSize, (unsigned)cSize,
ratioAccuracy, ratio,
benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT);
}
@@ -670,14 +461,7 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
}
if(!decompressionCompleted) {
- BMK_runOutcome_t const dOutcome =
- BMK_benchTimedFn(timeStateDecompress,
- &local_defaultDecompress, dctx,
- &local_initDCtx, &dctxprep,
- nbBlocks,
- (const void *const *)cPtrs, cSizes,
- resPtrs, resSizes,
- NULL);
+ BMK_runOutcome_t const dOutcome = BMK_benchTimedFn(timeStateDecompress, dbp);
if(!BMK_isSuccessful_runOutcome(dOutcome)) {
return BMK_benchOutcome_error();
@@ -690,16 +474,16 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
}
{ int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
- markNb = (markNb+1) % NB_MARKS;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
marks[markNb], displayName,
- (U32)srcSize, (U32)benchResult.cSize,
+ (unsigned)srcSize, (unsigned)benchResult.cSize,
ratioAccuracy, ratio,
benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT,
(double)benchResult.dSpeed / MB_UNIT);
}
decompressionCompleted = BMK_isCompleted_TimedFn(timeStateDecompress);
}
+ markNb = (markNb+1) % NB_MARKS;
} /* while (!(compressionCompleted && decompressionCompleted)) */
/* CRC Checking */
@@ -707,12 +491,13 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
if ((adv->mode == BMK_both) && (crcOrig!=crcCheck)) {
size_t u;
- DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
+ DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n",
+ displayName, (unsigned)crcOrig, (unsigned)crcCheck);
for (u=0; u<srcSize; u++) {
if (((const BYTE*)srcBuffer)[u] != resultBuffer[u]) {
- U32 segNb, bNb, pos;
+ unsigned segNb, bNb, pos;
size_t bacc = 0;
- DISPLAY("Decoding error at pos %u ", (U32)u);
+ DISPLAY("Decoding error at pos %u ", (unsigned)u);
for (segNb = 0; segNb < nbBlocks; segNb++) {
if (bacc + srcSizes[segNb] > u) break;
bacc += srcSizes[segNb];
@@ -883,7 +668,7 @@ static BMK_benchOutcome_t BMK_benchCLevel(const void* srcBuffer, size_t benchedS
if (displayLevel == 1 && !adv->additionalParam) /* --quiet mode */
DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n",
ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING,
- (U32)benchedSize, adv->nbSeconds, (U32)(adv->blockSize>>10));
+ (unsigned)benchedSize, adv->nbSeconds, (unsigned)(adv->blockSize>>10));
return BMK_benchMemAdvanced(srcBuffer, benchedSize,
NULL, 0,
@@ -1015,6 +800,11 @@ BMK_benchOutcome_t BMK_benchFilesAdvanced(
/* Load dictionary */
if (dictFileName != NULL) {
U64 const dictFileSize = UTIL_getFileSize(dictFileName);
+ if (dictFileSize == UTIL_FILESIZE_UNKNOWN) {
+ DISPLAYLEVEL(1, "error loading %s : %s \n", dictFileName, strerror(errno));
+ free(fileSizes);
+ RETURN_ERROR(9, BMK_benchOutcome_t, "benchmark aborted");
+ }
if (dictFileSize > 64 MB) {
free(fileSizes);
RETURN_ERROR(10, BMK_benchOutcome_t, "dictionary file %s too large", dictFileName);
@@ -1024,7 +814,7 @@ BMK_benchOutcome_t BMK_benchFilesAdvanced(
if (dictBuffer==NULL) {
free(fileSizes);
RETURN_ERROR(11, BMK_benchOutcome_t, "not enough memory for dictionary (%u bytes)",
- (U32)dictBufferSize);
+ (unsigned)dictBufferSize);
}
{ int const errorCode = BMK_loadFiles(dictBuffer, dictBufferSize,
@@ -1040,7 +830,7 @@ BMK_benchOutcome_t BMK_benchFilesAdvanced(
benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3;
if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
if (benchedSize < totalSizeToLoad)
- DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
+ DISPLAY("Not enough memory; testing %u MB only...\n", (unsigned)(benchedSize >> 20));
srcBuffer = benchedSize ? malloc(benchedSize) : NULL;
if (!srcBuffer) {
diff --git a/programs/bench.h b/programs/benchzstd.h
index 13ca5b50b4612..3a8b893e7ee60 100644
--- a/programs/bench.h
+++ b/programs/benchzstd.h
@@ -8,12 +8,18 @@
* You may select, at your option, one of the above-listed licenses.
*/
+ /* benchzstd :
+ * benchmark Zstandard compression / decompression
+ * over a set of files or buffers
+ * and display progress result and final summary
+ */
+
#if defined (__cplusplus)
extern "C" {
#endif
-#ifndef BENCH_H_121279284357
-#define BENCH_H_121279284357
+#ifndef BENCH_ZSTD_H_3242387
+#define BENCH_ZSTD_H_3242387
/* === Dependencies === */
#include <stddef.h> /* size_t */
@@ -109,7 +115,7 @@ typedef struct {
unsigned ldmMinMatch; /* below: parameters for long distance matching, see zstd.1.md */
unsigned ldmHashLog;
unsigned ldmBucketSizeLog;
- unsigned ldmHashEveryLog;
+ unsigned ldmHashRateLog;
} BMK_advancedParams_t;
/* returns default parameters used by nonAdvanced functions */
@@ -142,9 +148,9 @@ BMK_benchOutcome_t BMK_benchFilesAdvanced(
* .cMem : memory budget required for the compression context
*/
BMK_benchOutcome_t BMK_syntheticTest(
- int cLevel, double compressibility,
- const ZSTD_compressionParameters* compressionParams,
- int displayLevel, const BMK_advancedParams_t* adv);
+ int cLevel, double compressibility,
+ const ZSTD_compressionParameters* compressionParams,
+ int displayLevel, const BMK_advancedParams_t* adv);
@@ -181,6 +187,7 @@ BMK_benchOutcome_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
const void* dictBuffer, size_t dictBufferSize,
int displayLevel, const char* displayName);
+
/* BMK_benchMemAdvanced() : same as BMK_benchMem()
* with following additional options :
* dstBuffer - destination buffer to write compressed output in, NULL if none provided.
@@ -197,106 +204,8 @@ BMK_benchOutcome_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
-/* ==== Benchmarking any function, iterated on a set of blocks ==== */
-
-typedef struct {
- unsigned long long nanoSecPerRun; /* time per iteration */
- size_t sumOfReturn; /* sum of return values */
-} BMK_runTime_t;
-
-VARIANT_ERROR_RESULT(BMK_runTime_t, BMK_runOutcome_t);
-
-/* check first if the return structure represents an error or a valid result */
-int BMK_isSuccessful_runOutcome(BMK_runOutcome_t outcome);
-
-/* extract result from variant type.
- * note : this function will abort() program execution if result is not valid
- * check result validity first, by using BMK_isSuccessful_runOutcome()
- */
-BMK_runTime_t BMK_extract_runTime(BMK_runOutcome_t outcome);
-
-
-
-typedef size_t (*BMK_benchFn_t)(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* customPayload);
-typedef size_t (*BMK_initFn_t)(void* initPayload);
-
-
-/* BMK_benchFunction() :
- * This function times the execution of 2 argument functions, benchFn and initFn */
-
-/* benchFn - (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstCapacities[i], benchPayload)
- * is run nbLoops times
- * initFn - (*initFn)(initPayload) is run once per benchmark, at the beginning.
- * This argument can be NULL, in which case nothing is run.
- * blockCount - number of blocks. Size of all array parameters : srcBuffers, srcSizes, dstBuffers, dstCapacities, blockResults
- * srcBuffers - an array of buffers to be operated on by benchFn
- * srcSizes - an array of the sizes of above buffers
- * dstBuffers - an array of buffers to be written into by benchFn
- * dstCapacities - an array of the capacities of above buffers
- * blockResults - Optional: store the return value of benchFn for each block. Use NULL if this result is not requested.
- * nbLoops - defines number of times benchFn is run.
- * @return: a variant, which express either an error, or can generate a valid BMK_runTime_t result.
- * Use BMK_isSuccessful_runOutcome() to check if function was successful.
- * If yes, extract the result with BMK_extract_runTime(),
- * it will contain :
- * .sumOfReturn : the sum of all return values of benchFn through all of blocks
- * .nanoSecPerRun : time per run of benchFn + (time for initFn / nbLoops)
- * .sumOfReturn is generally intended for functions which return a # of bytes written into dstBuffer,
- * in which case, this value will be the total amount of bytes written into dstBuffer.
- */
-BMK_runOutcome_t BMK_benchFunction(
- BMK_benchFn_t benchFn, void* benchPayload,
- BMK_initFn_t initFn, void* initPayload,
- size_t blockCount,
- const void *const * srcBuffers, const size_t* srcSizes,
- void *const * dstBuffers, const size_t* dstCapacities,
- size_t* blockResults,
- unsigned nbLoops);
-
-
-
-/* ==== Benchmark any function, providing intermediate results ==== */
-
-/* state information tracking benchmark session */
-typedef struct BMK_timedFnState_s BMK_timedFnState_t;
-
-/* BMK_createTimedFnState() and BMK_resetTimedFnState() :
- * Create/Set BMK_timedFnState_t for next benchmark session,
- * which shall last a minimum of total_ms milliseconds,
- * producing intermediate results, paced at interval of (approximately) run_ms.
- */
-BMK_timedFnState_t* BMK_createTimedFnState(unsigned total_ms, unsigned run_ms);
-void BMK_resetTimedFnState(BMK_timedFnState_t* timedFnState, unsigned total_ms, unsigned run_ms);
-void BMK_freeTimedFnState(BMK_timedFnState_t* state);
-
-
-/* Tells if duration of all benchmark runs has exceeded total_ms
- */
-int BMK_isCompleted_TimedFn(const BMK_timedFnState_t* timedFnState);
-
-
-/* BMK_benchTimedFn() :
- * Similar to BMK_benchFunction(), most arguments being identical.
- * Automatically determines `nbLoops` so that each result is regularly produced at interval of about run_ms.
- * Note : minimum `nbLoops` is 1, therefore a run may last more than run_ms, and possibly even more than total_ms.
- * Usage - initialize timedFnState, select benchmark duration (total_ms) and each measurement duration (run_ms)
- * call BMK_benchTimedFn() repetitively, each measurement is supposed to last about run_ms
- * Check if total time budget is spent or exceeded, using BMK_isCompleted_TimedFn()
- */
-BMK_runOutcome_t BMK_benchTimedFn(
- BMK_timedFnState_t* timedFnState,
- BMK_benchFn_t benchFn, void* benchPayload,
- BMK_initFn_t initFn, void* initPayload,
- size_t blockCount,
- const void *const * srcBlockBuffers, const size_t* srcBlockSizes,
- void *const * dstBlockBuffers, const size_t* dstBlockCapacities,
- size_t* blockResults);
-
-
-
-
-#endif /* BENCH_H_121279284357 */
+#endif /* BENCH_ZSTD_H_3242387 */
#if defined (__cplusplus)
}
diff --git a/programs/datagen.c b/programs/datagen.c
index c8383658488b1..026b9a1855da0 100644
--- a/programs/datagen.c
+++ b/programs/datagen.c
@@ -81,18 +81,18 @@ static BYTE RDG_genChar(U32* seed, const BYTE* ldt)
}
-static U32 RDG_rand15Bits (unsigned* seedPtr)
+static U32 RDG_rand15Bits (U32* seedPtr)
{
return RDG_rand(seedPtr) & 0x7FFF;
}
-static U32 RDG_randLength(unsigned* seedPtr)
+static U32 RDG_randLength(U32* seedPtr)
{
if (RDG_rand(seedPtr) & 7) return (RDG_rand(seedPtr) & 0xF); /* small length */
return (RDG_rand(seedPtr) & 0x1FF) + 0xF;
}
-static void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, const BYTE* ldt, unsigned* seedPtr)
+static void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, const BYTE* ldt, U32* seedPtr)
{
BYTE* const buffPtr = (BYTE*)buffer;
U32 const matchProba32 = (U32)(32768 * matchProba);
@@ -141,16 +141,18 @@ static void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, doubl
void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed)
{
+ U32 seed32 = seed;
BYTE ldt[LTSIZE];
memset(ldt, '0', sizeof(ldt)); /* yes, character '0', this is intentional */
if (litProba<=0.0) litProba = matchProba / 4.5;
RDG_fillLiteralDistrib(ldt, litProba);
- RDG_genBlock(buffer, size, 0, matchProba, ldt, &seed);
+ RDG_genBlock(buffer, size, 0, matchProba, ldt, &seed32);
}
void RDG_genStdout(unsigned long long size, double matchProba, double litProba, unsigned seed)
{
+ U32 seed32 = seed;
size_t const stdBlockSize = 128 KB;
size_t const stdDictSize = 32 KB;
BYTE* const buff = (BYTE*)malloc(stdDictSize + stdBlockSize);
@@ -165,12 +167,12 @@ void RDG_genStdout(unsigned long long size, double matchProba, double litProba,
SET_BINARY_MODE(stdout);
/* Generate initial dict */
- RDG_genBlock(buff, stdDictSize, 0, matchProba, ldt, &seed);
+ RDG_genBlock(buff, stdDictSize, 0, matchProba, ldt, &seed32);
/* Generate compressible data */
while (total < size) {
size_t const genBlockSize = (size_t) (MIN (stdBlockSize, size-total));
- RDG_genBlock(buff, stdDictSize+stdBlockSize, stdDictSize, matchProba, ldt, &seed);
+ RDG_genBlock(buff, stdDictSize+stdBlockSize, stdDictSize, matchProba, ldt, &seed32);
total += genBlockSize;
{ size_t const unused = fwrite(buff, 1, genBlockSize, stdout); (void)unused; }
/* update dict */
diff --git a/programs/dibio.c b/programs/dibio.c
index d3fd8cc053de0..c9d214e754e61 100644
--- a/programs/dibio.c
+++ b/programs/dibio.c
@@ -139,7 +139,7 @@ static unsigned DiB_loadFiles(void* buffer, size_t* bufferSizePtr,
}
DISPLAYLEVEL(2, "\r%79s\r", "");
*bufferSizePtr = pos;
- DISPLAYLEVEL(4, "loaded : %u KB \n", (U32)(pos >> 10))
+ DISPLAYLEVEL(4, "loaded : %u KB \n", (unsigned)(pos >> 10))
return nbLoadedChunks;
}
@@ -249,7 +249,7 @@ static fileStats DiB_fileStats(const char** fileNamesTable, unsigned nbFiles, si
fs.oneSampleTooLarge |= (chunkSize > 2*SAMPLESIZE_MAX);
fs.nbSamples += nbSamples;
}
- DISPLAYLEVEL(4, "Preparing to load : %u KB \n", (U32)(fs.totalSizeToLoad >> 10));
+ DISPLAYLEVEL(4, "Preparing to load : %u KB \n", (unsigned)(fs.totalSizeToLoad >> 10));
return fs;
}
@@ -358,7 +358,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
goto _cleanup;
}
/* save dict */
- DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (U32)dictSize, dictFileName);
+ DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (unsigned)dictSize, dictFileName);
DiB_saveDict(dictFileName, dictBuffer, dictSize);
}
diff --git a/programs/fileio.c b/programs/fileio.c
index c24f4defbb9ad..9fb795ed3f27b 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -88,10 +88,10 @@ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
static const U64 g_refreshRate = SEC_TO_MICRO / 6;
static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
-#define READY_FOR_UPDATE() (UTIL_clockSpanMicro(g_displayClock) > g_refreshRate)
+#define READY_FOR_UPDATE() (!g_noProgress && UTIL_clockSpanMicro(g_displayClock) > g_refreshRate)
#define DELAY_NEXT_UPDATE() { g_displayClock = UTIL_getTime(); }
#define DISPLAYUPDATE(l, ...) { \
- if (g_displayLevel>=l) { \
+ if (g_displayLevel>=l && !g_noProgress) { \
if (READY_FOR_UPDATE() || (g_displayLevel>=4)) { \
DELAY_NEXT_UPDATE(); \
DISPLAY(__VA_ARGS__); \
@@ -270,18 +270,18 @@ void FIO_addAbortHandler()
static FIO_compressionType_t g_compressionType = FIO_zstdCompression;
void FIO_setCompressionType(FIO_compressionType_t compressionType) { g_compressionType = compressionType; }
static U32 g_overwrite = 0;
-void FIO_overwriteMode(void) { g_overwrite=1; }
-static U32 g_sparseFileSupport = 1; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
-void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; }
+void FIO_overwriteMode(void) { g_overwrite = 1; }
+static U32 g_sparseFileSupport = ZSTD_SPARSE_DEFAULT; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
+void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport = sparse; }
static U32 g_dictIDFlag = 1;
void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; }
static U32 g_checksumFlag = 1;
void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; }
static U32 g_removeSrcFile = 0;
void FIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); }
-static U32 g_memLimit = 0;
+static unsigned g_memLimit = 0;
void FIO_setMemLimit(unsigned memLimit) { g_memLimit = memLimit; }
-static U32 g_nbWorkers = 1;
+static unsigned g_nbWorkers = 1;
void FIO_setNbWorkers(unsigned nbWorkers) {
#ifndef ZSTD_MULTITHREAD
if (nbWorkers > 0) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n");
@@ -295,7 +295,7 @@ void FIO_setBlockSize(unsigned blockSize) {
g_blockSize = blockSize;
}
#define FIO_OVERLAP_LOG_NOTSET 9999
-static U32 g_overlapLog = FIO_OVERLAP_LOG_NOTSET;
+static unsigned g_overlapLog = FIO_OVERLAP_LOG_NOTSET;
void FIO_setOverlapLog(unsigned overlapLog){
if (overlapLog && g_nbWorkers==0)
DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n");
@@ -307,6 +307,12 @@ void FIO_setAdaptiveMode(unsigned adapt) {
EXM_THROW(1, "Adaptive mode is not compatible with single thread mode \n");
g_adaptiveMode = adapt;
}
+static U32 g_rsyncable = 0;
+void FIO_setRsyncable(unsigned rsyncable) {
+ if ((rsyncable>0) && (g_nbWorkers==0))
+ EXM_THROW(1, "Rsyncable mode is not compatible with single thread mode \n");
+ g_rsyncable = rsyncable;
+}
static int g_minAdaptLevel = -50; /* initializing this value requires a constant, so ZSTD_minCLevel() doesn't work */
void FIO_setAdaptMin(int minCLevel)
{
@@ -340,9 +346,13 @@ void FIO_setLdmBucketSizeLog(unsigned ldmBucketSizeLog) {
g_ldmBucketSizeLog = ldmBucketSizeLog;
}
-static U32 g_ldmHashEveryLog = FIO_LDM_PARAM_NOTSET;
-void FIO_setLdmHashEveryLog(unsigned ldmHashEveryLog) {
- g_ldmHashEveryLog = ldmHashEveryLog;
+static U32 g_ldmHashRateLog = FIO_LDM_PARAM_NOTSET;
+void FIO_setLdmHashRateLog(unsigned ldmHashRateLog) {
+ g_ldmHashRateLog = ldmHashRateLog;
+}
+static U32 g_noProgress = 0;
+void FIO_setNoProgress(unsigned noProgress) {
+ g_noProgress = noProgress;
}
@@ -355,7 +365,7 @@ void FIO_setLdmHashEveryLog(unsigned ldmHashEveryLog) {
static int FIO_remove(const char* path)
{
if (!UTIL_isRegularFile(path)) {
- DISPLAYLEVEL(2, "zstd: Refusing to remove non-regular file %s\n", path);
+ DISPLAYLEVEL(2, "zstd: Refusing to remove non-regular file %s \n", path);
return 0;
}
#if defined(_WIN32) || defined(WIN32)
@@ -373,11 +383,17 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
{
assert(srcFileName != NULL);
if (!strcmp (srcFileName, stdinmark)) {
- DISPLAYLEVEL(4,"Using stdin for input\n");
+ DISPLAYLEVEL(4,"Using stdin for input \n");
SET_BINARY_MODE(stdin);
return stdin;
}
+ if (!UTIL_fileExist(srcFileName)) {
+ DISPLAYLEVEL(1, "zstd: can't stat %s : %s -- ignored \n",
+ srcFileName, strerror(errno));
+ return NULL;
+ }
+
if (!UTIL_isRegularFile(srcFileName)) {
DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
srcFileName);
@@ -394,30 +410,54 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
/** FIO_openDstFile() :
* condition : `dstFileName` must be non-NULL.
* @result : FILE* to `dstFileName`, or NULL if it fails */
-static FILE* FIO_openDstFile(const char* dstFileName)
+static FILE* FIO_openDstFile(const char* srcFileName, const char* dstFileName)
{
assert(dstFileName != NULL);
if (!strcmp (dstFileName, stdoutmark)) {
- DISPLAYLEVEL(4,"Using stdout for output\n");
+ DISPLAYLEVEL(4,"Using stdout for output \n");
SET_BINARY_MODE(stdout);
- if (g_sparseFileSupport==1) {
+ if (g_sparseFileSupport == 1) {
g_sparseFileSupport = 0;
DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n");
}
return stdout;
}
+ /* ensure dst is not the same file as src */
+ if (srcFileName != NULL) {
+#ifdef _MSC_VER
+ /* note : Visual does not support file identification by inode.
+ * The following work-around is limited to detecting exact name repetition only,
+ * aka `filename` is considered different from `subdir/../filename` */
+ if (!strcmp(srcFileName, dstFileName)) {
+ DISPLAYLEVEL(1, "zstd: Refusing to open a output file which will overwrite the input file \n");
+ return NULL;
+ }
+#else
+ stat_t srcStat;
+ stat_t dstStat;
+ if (UTIL_getFileStat(srcFileName, &srcStat)
+ && UTIL_getFileStat(dstFileName, &dstStat)) {
+ if (srcStat.st_dev == dstStat.st_dev
+ && srcStat.st_ino == dstStat.st_ino) {
+ DISPLAYLEVEL(1, "zstd: Refusing to open a output file which will overwrite the input file \n");
+ return NULL;
+ }
+ }
+#endif
+ }
+
if (g_sparseFileSupport == 1) {
g_sparseFileSupport = ZSTD_SPARSE_DEFAULT;
}
if (UTIL_isRegularFile(dstFileName)) {
- FILE* fCheck;
+ /* Check if destination file already exists */
+ FILE* const fCheck = fopen( dstFileName, "rb" );
if (!strcmp(dstFileName, nulmark)) {
- EXM_THROW(40, "%s is unexpectedly a regular file", dstFileName);
+ EXM_THROW(40, "%s is unexpectedly categorized as a regular file",
+ dstFileName);
}
- /* Check if destination file already exists */
- fCheck = fopen( dstFileName, "rb" );
if (fCheck != NULL) { /* dst file exists, authorization prompt */
fclose(fCheck);
if (!g_overwrite) {
@@ -467,6 +507,7 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
fileHandle = fopen(fileName, "rb");
if (fileHandle==NULL) EXM_THROW(31, "%s: %s", fileName, strerror(errno));
+
fileSize = UTIL_getFileSize(fileName);
if (fileSize > DICTSIZE_MAX) {
EXM_THROW(32, "Dictionary file %s is too large (> %u MB)",
@@ -475,8 +516,9 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
*bufferPtr = malloc((size_t)fileSize);
if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno));
{ size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
- if (readSize!=fileSize)
- EXM_THROW(35, "Error reading dictionary file %s", fileName);
+ if (readSize != fileSize)
+ EXM_THROW(35, "Error reading dictionary file %s : %s",
+ fileName, strerror(errno));
}
fclose(fileHandle);
return (size_t)fileSize;
@@ -506,7 +548,8 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
DISPLAYLEVEL(6, "FIO_createCResources \n");
ress.cctx = ZSTD_createCCtx();
if (ress.cctx == NULL)
- EXM_THROW(30, "allocation error : can't create ZSTD_CCtx");
+ EXM_THROW(30, "allocation error (%s): can't create ZSTD_CCtx",
+ strerror(errno));
ress.srcBufferSize = ZSTD_CStreamInSize();
ress.srcBuffer = malloc(ress.srcBufferSize);
ress.dstBufferSize = ZSTD_CStreamOutSize();
@@ -523,40 +566,39 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
if (g_adaptiveMode && !g_ldmFlag && !comprParams.windowLog)
comprParams.windowLog = ADAPT_WINDOWLOG_DEFAULT;
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) ); /* always enable content size when available (note: supposed to be default) */
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_contentSizeFlag, 1) ); /* always enable content size when available (note: supposed to be default) */
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_dictIDFlag, g_dictIDFlag) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_checksumFlag, g_checksumFlag) );
/* compression level */
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)cLevel) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, cLevel) );
/* long distance matching */
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashLog, g_ldmHashLog) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmMinMatch, g_ldmMinMatch) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_enableLongDistanceMatching, g_ldmFlag) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashLog, g_ldmHashLog) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmMinMatch, g_ldmMinMatch) );
if (g_ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) {
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmBucketSizeLog, g_ldmBucketSizeLog) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmBucketSizeLog, g_ldmBucketSizeLog) );
}
- if (g_ldmHashEveryLog != FIO_LDM_PARAM_NOTSET) {
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashEveryLog, g_ldmHashEveryLog) );
+ if (g_ldmHashRateLog != FIO_LDM_PARAM_NOTSET) {
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashRateLog, g_ldmHashRateLog) );
}
/* compression parameters */
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_windowLog, comprParams.windowLog) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_chainLog, comprParams.chainLog) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_hashLog, comprParams.hashLog) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_searchLog, comprParams.searchLog) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_minMatch, comprParams.searchLength) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_targetLength, comprParams.targetLength) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionStrategy, (U32)comprParams.strategy) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_windowLog, comprParams.windowLog) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_chainLog, comprParams.chainLog) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_hashLog, comprParams.hashLog) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_searchLog, comprParams.searchLog) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_minMatch, comprParams.minMatch) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetLength, comprParams.targetLength) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_strategy, comprParams.strategy) );
/* multi-threading */
#ifdef ZSTD_MULTITHREAD
DISPLAYLEVEL(5,"set nb workers = %u \n", g_nbWorkers);
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbWorkers, g_nbWorkers) );
- if ( (g_overlapLog == FIO_OVERLAP_LOG_NOTSET)
- && (cLevel == ZSTD_maxCLevel()) )
- g_overlapLog = 9; /* full overlap */
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_nbWorkers, g_nbWorkers) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, g_blockSize) );
if (g_overlapLog != FIO_OVERLAP_LOG_NOTSET) {
DISPLAYLEVEL(3,"set overlapLog = %u \n", g_overlapLog);
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_overlapSizeLog, g_overlapLog) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_overlapLog, g_overlapLog) );
}
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_rsyncable, g_rsyncable) );
#endif
/* dictionary */
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* set the value temporarily for dictionary loading, to adapt compression parameters */
@@ -627,11 +669,11 @@ FIO_compressGzFrame(cRess_t* ress,
}
if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
- (U32)(inFileSize>>20),
+ (unsigned)(inFileSize>>20),
(double)outFileSize/inFileSize*100)
else
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
- (U32)(inFileSize>>20), (U32)(srcFileSize>>20),
+ (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
(double)outFileSize/inFileSize*100);
}
@@ -640,7 +682,7 @@ FIO_compressGzFrame(cRess_t* ress,
{ size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
if (decompBytes) {
if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes)
- EXM_THROW(75, "Write error : cannot write to output file");
+ EXM_THROW(75, "Write error : %s", strerror(errno));
outFileSize += decompBytes;
strm.next_out = (Bytef*)ress->dstBuffer;
strm.avail_out = (uInt)ress->dstBufferSize;
@@ -708,18 +750,18 @@ FIO_compressLzmaFrame(cRess_t* ress,
{ size_t const compBytes = ress->dstBufferSize - strm.avail_out;
if (compBytes) {
if (fwrite(ress->dstBuffer, 1, compBytes, ress->dstFile) != compBytes)
- EXM_THROW(73, "Write error : cannot write to output file");
+ EXM_THROW(73, "Write error : %s", strerror(errno));
outFileSize += compBytes;
strm.next_out = (BYTE*)ress->dstBuffer;
strm.avail_out = ress->dstBufferSize;
} }
if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
- (U32)(inFileSize>>20),
+ (unsigned)(inFileSize>>20),
(double)outFileSize/inFileSize*100)
else
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
- (U32)(inFileSize>>20), (U32)(srcFileSize>>20),
+ (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
(double)outFileSize/inFileSize*100);
if (ret == LZMA_STREAM_END) break;
}
@@ -773,7 +815,7 @@ FIO_compressLz4Frame(cRess_t* ress,
EXM_THROW(33, "File header generation failed : %s",
LZ4F_getErrorName(headerSize));
if (fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile) != headerSize)
- EXM_THROW(34, "Write error : cannot write header");
+ EXM_THROW(34, "Write error : %s (cannot write header)", strerror(errno));
outFileSize += headerSize;
/* Read first block */
@@ -782,26 +824,28 @@ FIO_compressLz4Frame(cRess_t* ress,
/* Main Loop */
while (readSize>0) {
- size_t outSize;
-
- /* Compress Block */
- outSize = LZ4F_compressUpdate(ctx, ress->dstBuffer, ress->dstBufferSize, ress->srcBuffer, readSize, NULL);
+ size_t const outSize = LZ4F_compressUpdate(ctx,
+ ress->dstBuffer, ress->dstBufferSize,
+ ress->srcBuffer, readSize, NULL);
if (LZ4F_isError(outSize))
EXM_THROW(35, "zstd: %s: lz4 compression failed : %s",
srcFileName, LZ4F_getErrorName(outSize));
outFileSize += outSize;
- if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
+ if (srcFileSize == UTIL_FILESIZE_UNKNOWN) {
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
- (U32)(inFileSize>>20),
+ (unsigned)(inFileSize>>20),
(double)outFileSize/inFileSize*100)
- else
+ } else {
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
- (U32)(inFileSize>>20), (U32)(srcFileSize>>20),
+ (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
(double)outFileSize/inFileSize*100);
+ }
/* Write Block */
- { size_t const sizeCheck = fwrite(ress->dstBuffer, 1, outSize, ress->dstFile);
- if (sizeCheck!=outSize) EXM_THROW(36, "Write error : cannot write compressed block"); }
+ { size_t const sizeCheck = fwrite(ress->dstBuffer, 1, outSize, ress->dstFile);
+ if (sizeCheck != outSize)
+ EXM_THROW(36, "Write error : %s", strerror(errno));
+ }
/* Read next block */
readSize = fread(ress->srcBuffer, (size_t)1, (size_t)blockSize, ress->srcFile);
@@ -815,8 +859,11 @@ FIO_compressLz4Frame(cRess_t* ress,
EXM_THROW(38, "zstd: %s: lz4 end of file generation failed : %s",
srcFileName, LZ4F_getErrorName(headerSize));
- { size_t const sizeCheck = fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile);
- if (sizeCheck!=headerSize) EXM_THROW(39, "Write error : cannot write end of stream"); }
+ { size_t const sizeCheck = fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile);
+ if (sizeCheck != headerSize)
+ EXM_THROW(39, "Write error : %s (cannot write end of stream)",
+ strerror(errno));
+ }
outFileSize += headerSize;
}
@@ -863,7 +910,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
/* Fill input Buffer */
size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile);
ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 };
- DISPLAYLEVEL(6, "fread %u bytes from source \n", (U32)inSize);
+ DISPLAYLEVEL(6, "fread %u bytes from source \n", (unsigned)inSize);
*readsize += inSize;
if ((inSize == 0) || (*readsize == fileSize))
@@ -876,7 +923,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
size_t const oldIPos = inBuff.pos;
ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
size_t const toFlushNow = ZSTD_toFlushNow(ress.cctx);
- CHECK_V(stillToFlush, ZSTD_compress_generic(ress.cctx, &outBuff, &inBuff, directive));
+ CHECK_V(stillToFlush, ZSTD_compressStream2(ress.cctx, &outBuff, &inBuff, directive));
/* count stats */
inputPresented++;
@@ -885,11 +932,12 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
/* Write compressed stream */
DISPLAYLEVEL(6, "ZSTD_compress_generic(end:%u) => input pos(%u)<=(%u)size ; output generated %u bytes \n",
- (U32)directive, (U32)inBuff.pos, (U32)inBuff.size, (U32)outBuff.pos);
+ (unsigned)directive, (unsigned)inBuff.pos, (unsigned)inBuff.size, (unsigned)outBuff.pos);
if (outBuff.pos) {
size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
if (sizeCheck != outBuff.pos)
- EXM_THROW(25, "Write error : cannot write compressed block");
+ EXM_THROW(25, "Write error : %s (cannot write compressed block)",
+ strerror(errno));
compressedfilesize += outBuff.pos;
}
@@ -902,14 +950,14 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
if (g_displayLevel >= 3) {
DISPLAYUPDATE(3, "\r(L%i) Buffered :%4u MB - Consumed :%4u MB - Compressed :%4u MB => %.2f%% ",
compressionLevel,
- (U32)((zfp.ingested - zfp.consumed) >> 20),
- (U32)(zfp.consumed >> 20),
- (U32)(zfp.produced >> 20),
+ (unsigned)((zfp.ingested - zfp.consumed) >> 20),
+ (unsigned)(zfp.consumed >> 20),
+ (unsigned)(zfp.produced >> 20),
cShare );
} else { /* summarized notifications if == 2; */
- DISPLAYLEVEL(2, "\rRead : %u ", (U32)(zfp.consumed >> 20));
+ DISPLAYLEVEL(2, "\rRead : %u ", (unsigned)(zfp.consumed >> 20));
if (fileSize != UTIL_FILESIZE_UNKNOWN)
- DISPLAYLEVEL(2, "/ %u ", (U32)(fileSize >> 20));
+ DISPLAYLEVEL(2, "/ %u ", (unsigned)(fileSize >> 20));
DISPLAYLEVEL(2, "MB ==> %2.f%% ", cShare);
DELAY_NEXT_UPDATE();
}
@@ -965,8 +1013,8 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
assert(inputPresented > 0);
DISPLAYLEVEL(6, "input blocked %u/%u(%.2f) - ingested:%u vs %u:consumed - flushed:%u vs %u:produced \n",
inputBlocked, inputPresented, (double)inputBlocked/inputPresented*100,
- (U32)newlyIngested, (U32)newlyConsumed,
- (U32)newlyFlushed, (U32)newlyProduced);
+ (unsigned)newlyIngested, (unsigned)newlyConsumed,
+ (unsigned)newlyFlushed, (unsigned)newlyProduced);
if ( (inputBlocked > inputPresented / 8) /* input is waiting often, because input buffers is full : compression or output too slow */
&& (newlyFlushed * 33 / 32 > newlyProduced) /* flush everything that is produced */
&& (newlyIngested * 33 / 32 > newlyConsumed) /* input speed as fast or faster than compression speed */
@@ -986,14 +1034,14 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel();
if (compressionLevel > g_maxAdaptLevel) compressionLevel = g_maxAdaptLevel;
compressionLevel += (compressionLevel == 0); /* skip 0 */
- ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)compressionLevel);
+ ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel);
}
if (speedChange == faster) {
DISPLAYLEVEL(6, "faster speed , lighter compression \n")
compressionLevel --;
if (compressionLevel < g_minAdaptLevel) compressionLevel = g_minAdaptLevel;
compressionLevel -= (compressionLevel == 0); /* skip 0 */
- ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, (unsigned)compressionLevel);
+ ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel);
}
speedChange = noChange;
@@ -1028,7 +1076,7 @@ FIO_compressFilename_internal(cRess_t ress,
U64 readsize = 0;
U64 compressedfilesize = 0;
U64 const fileSize = UTIL_getFileSize(srcFileName);
- DISPLAYLEVEL(5, "%s: %u bytes \n", srcFileName, (U32)fileSize);
+ DISPLAYLEVEL(5, "%s: %u bytes \n", srcFileName, (unsigned)fileSize);
/* compression format selection */
switch (g_compressionType) {
@@ -1105,7 +1153,7 @@ static int FIO_compressFilename_dstFile(cRess_t ress,
if (ress.dstFile == NULL) {
closeDstFile = 1;
DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s", dstFileName);
- ress.dstFile = FIO_openDstFile(dstFileName);
+ ress.dstFile = FIO_openDstFile(srcFileName, dstFileName);
if (ress.dstFile==NULL) return 1; /* could not open dstFileName */
/* Must only be added after FIO_openDstFile() succeeds.
* Otherwise we may delete the destination file if it already exists,
@@ -1255,7 +1303,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
assert(outFileName != NULL || suffix != NULL);
if (outFileName != NULL) { /* output into a single destination (stdout typically) */
- ress.dstFile = FIO_openDstFile(outFileName);
+ ress.dstFile = FIO_openDstFile(NULL, outFileName);
if (ress.dstFile == NULL) { /* could not open outFileName */
error = 1;
} else {
@@ -1263,7 +1311,8 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
for (u=0; u<nbFiles; u++)
error |= FIO_compressFilename_srcFile(ress, outFileName, inFileNamesTable[u], compressionLevel);
if (fclose(ress.dstFile))
- EXM_THROW(29, "Write error : cannot properly close %s", outFileName);
+ EXM_THROW(29, "Write error (%s) : cannot properly close %s",
+ strerror(errno), outFileName);
ress.dstFile = NULL;
}
} else {
@@ -1304,8 +1353,9 @@ static dRess_t FIO_createDResources(const char* dictFileName)
/* Allocation */
ress.dctx = ZSTD_createDStream();
- if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZSTD_DStream");
- CHECK( ZSTD_setDStreamParameter(ress.dctx, DStream_p_maxWindowSize, g_memLimit) );
+ if (ress.dctx==NULL)
+ EXM_THROW(60, "Error: %s : can't create ZSTD_DStream", strerror(errno));
+ CHECK( ZSTD_DCtx_setMaxWindowSize(ress.dctx, g_memLimit) );
ress.srcBufferSize = ZSTD_DStreamInSize();
ress.srcBuffer = malloc(ress.srcBufferSize);
ress.dstBufferSize = ZSTD_DStreamOutSize();
@@ -1343,14 +1393,17 @@ static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSi
if (!g_sparseFileSupport) { /* normal write */
size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file);
- if (sizeCheck != bufferSize) EXM_THROW(70, "Write error : cannot write decoded block");
+ if (sizeCheck != bufferSize)
+ EXM_THROW(70, "Write error : %s (cannot write decoded block)",
+ strerror(errno));
return 0;
}
/* avoid int overflow */
if (storedSkips > 1 GB) {
int const seekResult = LONG_SEEK(file, 1 GB, SEEK_CUR);
- if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)");
+ if (seekResult != 0)
+ EXM_THROW(71, "1 GB skip error (sparse file support)");
storedSkips -= 1 GB;
}
@@ -1401,12 +1454,14 @@ static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSi
static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
{
- if (storedSkips-->0) { /* implies g_sparseFileSupport>0 */
- int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
- if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)");
+ if (storedSkips>0) {
+ assert(g_sparseFileSupport > 0); /* storedSkips>0 implies sparse support is enabled */
+ if (LONG_SEEK(file, storedSkips-1, SEEK_CUR) != 0)
+ EXM_THROW(69, "Final skip error (sparse file support)");
+ /* last zero must be explicitly written,
+ * so that skipped ones get implicitly translated as zero by FS */
{ const char lastZeroByte[1] = { 0 };
- size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file);
- if (sizeCheck != 1)
+ if (fwrite(lastZeroByte, 1, 1, file) != 1)
EXM_THROW(69, "Write error : cannot write last zero");
} }
}
@@ -1463,12 +1518,12 @@ static void FIO_zstdErrorHelp(dRess_t* ress, size_t err, char const* srcFileName
err = ZSTD_getFrameHeader(&header, ress->srcBuffer, ress->srcBufferLoaded);
if (err == 0) {
unsigned long long const windowSize = header.windowSize;
- U32 const windowLog = FIO_highbit64(windowSize) + ((windowSize & (windowSize - 1)) != 0);
+ unsigned const windowLog = FIO_highbit64(windowSize) + ((windowSize & (windowSize - 1)) != 0);
assert(g_memLimit > 0);
DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u\n",
srcFileName, windowSize, g_memLimit);
if (windowLog <= ZSTD_WINDOWLOG_MAX) {
- U32 const windowMB = (U32)((windowSize >> 20) + ((windowSize & ((1 MB) - 1)) != 0));
+ unsigned const windowMB = (unsigned)((windowSize >> 20) + ((windowSize & ((1 MB) - 1)) != 0));
assert(windowSize < (U64)(1ULL << 52)); /* ensure now overflow for windowMB */
DISPLAYLEVEL(1, "%s : Use --long=%u or --memory=%uMB\n",
srcFileName, windowLog, windowMB);
@@ -1520,7 +1575,7 @@ static unsigned long long FIO_decompressZstdFrame(dRess_t* ress,
storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, storedSkips);
frameSize += outBuff.pos;
DISPLAYUPDATE(2, "\r%-20.20s : %u MB... ",
- srcFileName, (U32)((alreadyDecoded+frameSize)>>20) );
+ srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) );
if (inBuff.pos > 0) {
memmove(ress->srcBuffer, (char*)ress->srcBuffer + inBuff.pos, inBuff.size - inBuff.pos);
@@ -1871,7 +1926,7 @@ static int FIO_decompressDstFile(dRess_t ress, FILE* srcFile,
if (ress.dstFile == NULL) {
releaseDstFile = 1;
- ress.dstFile = FIO_openDstFile(dstFileName);
+ ress.dstFile = FIO_openDstFile(srcFileName, dstFileName);
if (ress.dstFile==0) return 1;
/* Must only be added after FIO_openDstFile() succeeds.
@@ -2025,7 +2080,7 @@ FIO_determineDstName(const char* srcFileName)
dfnbCapacity = sfnSize + 20;
dstFileNameBuffer = (char*)malloc(dfnbCapacity);
if (dstFileNameBuffer==NULL)
- EXM_THROW(74, "not enough memory for dstFileName");
+ EXM_THROW(74, "%s : not enough memory for dstFileName", strerror(errno));
}
/* return dst name == src name truncated from suffix */
@@ -2048,12 +2103,13 @@ FIO_decompressMultipleFilenames(const char* srcNamesTable[], unsigned nbFiles,
if (outFileName) {
unsigned u;
- ress.dstFile = FIO_openDstFile(outFileName);
+ ress.dstFile = FIO_openDstFile(NULL, outFileName);
if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", outFileName);
for (u=0; u<nbFiles; u++)
error |= FIO_decompressSrcFile(ress, outFileName, srcNamesTable[u]);
if (fclose(ress.dstFile))
- EXM_THROW(72, "Write error : cannot properly close output file");
+ EXM_THROW(72, "Write error : %s : cannot properly close output file",
+ strerror(errno));
} else {
unsigned u;
for (u=0; u<nbFiles; u++) { /* create dstFileName */
@@ -2103,7 +2159,7 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
for ( ; ; ) {
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile);
- if (numBytesRead < ZSTD_frameHeaderSize_min) {
+ if (numBytesRead < ZSTD_FRAMEHEADERSIZE_MIN) {
if ( feof(srcFile)
&& (numBytesRead == 0)
&& (info->compressedSize > 0)
@@ -2164,7 +2220,7 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
info->numActualFrames++;
}
/* Skippable frame */
- else if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
+ else if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
U32 const frameSize = MEM_readLE32(headerBuffer + 4);
long const seek = (long)(8 + frameSize - numBytesRead);
ERROR_IF(LONG_SEEK(srcFile, seek, SEEK_CUR) != 0,
@@ -2292,7 +2348,7 @@ FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel)
}
displayInfo(inFileName, &info, displayLevel);
*total = FIO_addFInfo(*total, info);
- assert(error>=0 || error<=1);
+ assert(error == info_success || error == info_frame_error);
return error;
}
}
@@ -2339,13 +2395,13 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
total.numSkippableFrames + total.numActualFrames,
total.numSkippableFrames,
compressedSizeUnit, unitStr,
- checkString, total.nbFiles);
+ checkString, (unsigned)total.nbFiles);
} else {
DISPLAYOUT("%6d %5d %7.2f %2s %9.2f %2s %5.3f %5s %u files\n",
total.numSkippableFrames + total.numActualFrames,
total.numSkippableFrames,
compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr,
- ratio, checkString, total.nbFiles);
+ ratio, checkString, (unsigned)total.nbFiles);
} }
return error;
}
diff --git a/programs/fileio.h b/programs/fileio.h
index 4c7049cb7167f..97f2706320717 100644
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -56,7 +56,7 @@ void FIO_setChecksumFlag(unsigned checksumFlag);
void FIO_setDictIDFlag(unsigned dictIDFlag);
void FIO_setLdmBucketSizeLog(unsigned ldmBucketSizeLog);
void FIO_setLdmFlag(unsigned ldmFlag);
-void FIO_setLdmHashEveryLog(unsigned ldmHashEveryLog);
+void FIO_setLdmHashRateLog(unsigned ldmHashRateLog);
void FIO_setLdmHashLog(unsigned ldmHashLog);
void FIO_setLdmMinMatch(unsigned ldmMinMatch);
void FIO_setMemLimit(unsigned memLimit);
@@ -65,6 +65,8 @@ void FIO_setNotificationLevel(unsigned level);
void FIO_setOverlapLog(unsigned overlapLog);
void FIO_setRemoveSrcFile(unsigned flag);
void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
+void FIO_setRsyncable(unsigned rsyncable);
+void FIO_setNoProgress(unsigned noProgress);
/*-*************************************
diff --git a/programs/platform.h b/programs/platform.h
index 155ebcd1eb9c8..1a8f97bc4dff0 100644
--- a/programs/platform.h
+++ b/programs/platform.h
@@ -26,6 +26,7 @@ extern "C" {
# define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before <io.h> and <windows.h> */
# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */
# endif
+# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
#endif
diff --git a/programs/util.c b/programs/util.c
new file mode 100644
index 0000000000000..34634318c638a
--- /dev/null
+++ b/programs/util.c
@@ -0,0 +1,673 @@
+/*
+ * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+/*-****************************************
+* Dependencies
+******************************************/
+#include "util.h" /* note : ensure that platform.h is included first ! */
+#include <errno.h>
+#include <assert.h>
+
+
+int UTIL_fileExist(const char* filename)
+{
+ stat_t statbuf;
+#if defined(_MSC_VER)
+ int const stat_error = _stat64(filename, &statbuf);
+#else
+ int const stat_error = stat(filename, &statbuf);
+#endif
+ return !stat_error;
+}
+
+int UTIL_isRegularFile(const char* infilename)
+{
+ stat_t statbuf;
+ return UTIL_getFileStat(infilename, &statbuf); /* Only need to know whether it is a regular file */
+}
+
+int UTIL_getFileStat(const char* infilename, stat_t *statbuf)
+{
+ int r;
+#if defined(_MSC_VER)
+ r = _stat64(infilename, statbuf);
+ if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */
+#else
+ r = stat(infilename, statbuf);
+ if (r || !S_ISREG(statbuf->st_mode)) return 0; /* No good... */
+#endif
+ return 1;
+}
+
+int UTIL_setFileStat(const char *filename, stat_t *statbuf)
+{
+ int res = 0;
+ struct utimbuf timebuf;
+
+ if (!UTIL_isRegularFile(filename))
+ return -1;
+
+ timebuf.actime = time(NULL);
+ timebuf.modtime = statbuf->st_mtime;
+ res += utime(filename, &timebuf); /* set access and modification times */
+
+#if !defined(_WIN32)
+ res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */
+#endif
+
+ res += chmod(filename, statbuf->st_mode & 07777); /* Copy file permissions */
+
+ errno = 0;
+ return -res; /* number of errors is returned */
+}
+
+U32 UTIL_isDirectory(const char* infilename)
+{
+ int r;
+ stat_t statbuf;
+#if defined(_MSC_VER)
+ r = _stat64(infilename, &statbuf);
+ if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
+#else
+ r = stat(infilename, &statbuf);
+ if (!r && S_ISDIR(statbuf.st_mode)) return 1;
+#endif
+ return 0;
+}
+
+U32 UTIL_isLink(const char* infilename)
+{
+/* macro guards, as defined in : https://linux.die.net/man/2/lstat */
+#ifndef __STRICT_ANSI__
+#if defined(_BSD_SOURCE) \
+ || (defined(_XOPEN_SOURCE) && (_XOPEN_SOURCE >= 500)) \
+ || (defined(_XOPEN_SOURCE) && defined(_XOPEN_SOURCE_EXTENDED)) \
+ || (defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) \
+ || (defined(__APPLE__) && defined(__MACH__))
+ int r;
+ stat_t statbuf;
+ r = lstat(infilename, &statbuf);
+ if (!r && S_ISLNK(statbuf.st_mode)) return 1;
+#endif
+#endif
+ (void)infilename;
+ return 0;
+}
+
+U64 UTIL_getFileSize(const char* infilename)
+{
+ if (!UTIL_isRegularFile(infilename)) return UTIL_FILESIZE_UNKNOWN;
+ { int r;
+#if defined(_MSC_VER)
+ struct __stat64 statbuf;
+ r = _stat64(infilename, &statbuf);
+ if (r || !(statbuf.st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN;
+#elif defined(__MINGW32__) && defined (__MSVCRT__)
+ struct _stati64 statbuf;
+ r = _stati64(infilename, &statbuf);
+ if (r || !(statbuf.st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN;
+#else
+ struct stat statbuf;
+ r = stat(infilename, &statbuf);
+ if (r || !S_ISREG(statbuf.st_mode)) return UTIL_FILESIZE_UNKNOWN;
+#endif
+ return (U64)statbuf.st_size;
+ }
+}
+
+
+U64 UTIL_getTotalFileSize(const char* const * const fileNamesTable, unsigned nbFiles)
+{
+ U64 total = 0;
+ int error = 0;
+ unsigned n;
+ for (n=0; n<nbFiles; n++) {
+ U64 const size = UTIL_getFileSize(fileNamesTable[n]);
+ error |= (size == UTIL_FILESIZE_UNKNOWN);
+ total += size;
+ }
+ return error ? UTIL_FILESIZE_UNKNOWN : total;
+}
+
+#ifdef _WIN32
+int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
+{
+ char* path;
+ int dirLength, fnameLength, pathLength, nbFiles = 0;
+ WIN32_FIND_DATAA cFile;
+ HANDLE hFile;
+
+ dirLength = (int)strlen(dirName);
+ path = (char*) malloc(dirLength + 3);
+ if (!path) return 0;
+
+ memcpy(path, dirName, dirLength);
+ path[dirLength] = '\\';
+ path[dirLength+1] = '*';
+ path[dirLength+2] = 0;
+
+ hFile=FindFirstFileA(path, &cFile);
+ if (hFile == INVALID_HANDLE_VALUE) {
+ UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s'\n", dirName);
+ return 0;
+ }
+ free(path);
+
+ do {
+ fnameLength = (int)strlen(cFile.cFileName);
+ path = (char*) malloc(dirLength + fnameLength + 2);
+ if (!path) { FindClose(hFile); return 0; }
+ memcpy(path, dirName, dirLength);
+ path[dirLength] = '\\';
+ memcpy(path+dirLength+1, cFile.cFileName, fnameLength);
+ pathLength = dirLength+1+fnameLength;
+ path[pathLength] = 0;
+ if (cFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
+ if ( strcmp (cFile.cFileName, "..") == 0
+ || strcmp (cFile.cFileName, ".") == 0 )
+ continue;
+ /* Recursively call "UTIL_prepareFileList" with the new path. */
+ nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks);
+ if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
+ } else if ( (cFile.dwFileAttributes & FILE_ATTRIBUTE_NORMAL)
+ || (cFile.dwFileAttributes & FILE_ATTRIBUTE_ARCHIVE)
+ || (cFile.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED) ) {
+ if (*bufStart + *pos + pathLength >= *bufEnd) {
+ ptrdiff_t const newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
+ *bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
+ if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
+ *bufEnd = *bufStart + newListSize;
+ }
+ if (*bufStart + *pos + pathLength < *bufEnd) {
+ memcpy(*bufStart + *pos, path, pathLength+1 /* include final \0 */);
+ *pos += pathLength + 1;
+ nbFiles++;
+ }
+ }
+ free(path);
+ } while (FindNextFileA(hFile, &cFile));
+
+ FindClose(hFile);
+ return nbFiles;
+}
+
+#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */
+
+int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
+{
+ DIR *dir;
+ struct dirent *entry;
+ char* path;
+ int dirLength, fnameLength, pathLength, nbFiles = 0;
+
+ if (!(dir = opendir(dirName))) {
+ UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno));
+ return 0;
+ }
+
+ dirLength = (int)strlen(dirName);
+ errno = 0;
+ while ((entry = readdir(dir)) != NULL) {
+ if (strcmp (entry->d_name, "..") == 0 ||
+ strcmp (entry->d_name, ".") == 0) continue;
+ fnameLength = (int)strlen(entry->d_name);
+ path = (char*) malloc(dirLength + fnameLength + 2);
+ if (!path) { closedir(dir); return 0; }
+ memcpy(path, dirName, dirLength);
+
+ path[dirLength] = '/';
+ memcpy(path+dirLength+1, entry->d_name, fnameLength);
+ pathLength = dirLength+1+fnameLength;
+ path[pathLength] = 0;
+
+ if (!followLinks && UTIL_isLink(path)) {
+ UTIL_DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", path);
+ continue;
+ }
+
+ if (UTIL_isDirectory(path)) {
+ nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */
+ if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
+ } else {
+ if (*bufStart + *pos + pathLength >= *bufEnd) {
+ ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
+ *bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
+ *bufEnd = *bufStart + newListSize;
+ if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
+ }
+ if (*bufStart + *pos + pathLength < *bufEnd) {
+ memcpy(*bufStart + *pos, path, pathLength + 1); /* with final \0 */
+ *pos += pathLength + 1;
+ nbFiles++;
+ }
+ }
+ free(path);
+ errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */
+ }
+
+ if (errno != 0) {
+ UTIL_DISPLAYLEVEL(1, "readdir(%s) error: %s\n", dirName, strerror(errno));
+ free(*bufStart);
+ *bufStart = NULL;
+ }
+ closedir(dir);
+ return nbFiles;
+}
+
+#else
+
+int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
+{
+ (void)bufStart; (void)bufEnd; (void)pos; (void)followLinks;
+ UTIL_DISPLAYLEVEL(1, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName);
+ return 0;
+}
+
+#endif /* #ifdef _WIN32 */
+
+/*
+ * UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories,
+ * and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb).
+ * After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer)
+ * In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called.
+ */
+const char**
+UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
+ char** allocatedBuffer, unsigned* allocatedNamesNb,
+ int followLinks)
+{
+ size_t pos;
+ unsigned i, nbFiles;
+ char* buf = (char*)malloc(LIST_SIZE_INCREASE);
+ char* bufend = buf + LIST_SIZE_INCREASE;
+ const char** fileTable;
+
+ if (!buf) return NULL;
+
+ for (i=0, pos=0, nbFiles=0; i<inputNamesNb; i++) {
+ if (!UTIL_isDirectory(inputNames[i])) {
+ size_t const len = strlen(inputNames[i]);
+ if (buf + pos + len >= bufend) {
+ ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
+ buf = (char*)UTIL_realloc(buf, newListSize);
+ bufend = buf + newListSize;
+ if (!buf) return NULL;
+ }
+ if (buf + pos + len < bufend) {
+ memcpy(buf+pos, inputNames[i], len+1); /* with final \0 */
+ pos += len + 1;
+ nbFiles++;
+ }
+ } else {
+ nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks);
+ if (buf == NULL) return NULL;
+ } }
+
+ if (nbFiles == 0) { free(buf); return NULL; }
+
+ fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*));
+ if (!fileTable) { free(buf); return NULL; }
+
+ for (i=0, pos=0; i<nbFiles; i++) {
+ fileTable[i] = buf + pos;
+ pos += strlen(fileTable[i]) + 1;
+ }
+
+ if (buf + pos > bufend) { free(buf); free((void*)fileTable); return NULL; }
+
+ *allocatedBuffer = buf;
+ *allocatedNamesNb = nbFiles;
+
+ return fileTable;
+}
+
+/*-****************************************
+* Console log
+******************************************/
+int g_utilDisplayLevel;
+
+
+/*-****************************************
+* Time functions
+******************************************/
+#if defined(_WIN32) /* Windows */
+
+UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; }
+
+U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
+{
+ static LARGE_INTEGER ticksPerSecond;
+ static int init = 0;
+ if (!init) {
+ if (!QueryPerformanceFrequency(&ticksPerSecond))
+ UTIL_DISPLAYLEVEL(1, "ERROR: QueryPerformanceFrequency() failure\n");
+ init = 1;
+ }
+ return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
+}
+
+U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
+{
+ static LARGE_INTEGER ticksPerSecond;
+ static int init = 0;
+ if (!init) {
+ if (!QueryPerformanceFrequency(&ticksPerSecond))
+ UTIL_DISPLAYLEVEL(1, "ERROR: QueryPerformanceFrequency() failure\n");
+ init = 1;
+ }
+ return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
+}
+
+#elif defined(__APPLE__) && defined(__MACH__)
+
+UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); }
+
+U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
+{
+ static mach_timebase_info_data_t rate;
+ static int init = 0;
+ if (!init) {
+ mach_timebase_info(&rate);
+ init = 1;
+ }
+ return (((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom))/1000ULL;
+}
+
+U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
+{
+ static mach_timebase_info_data_t rate;
+ static int init = 0;
+ if (!init) {
+ mach_timebase_info(&rate);
+ init = 1;
+ }
+ return ((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom);
+}
+
+#elif (PLATFORM_POSIX_VERSION >= 200112L) \
+ && (defined(__UCLIBC__) \
+ || (defined(__GLIBC__) \
+ && ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 17) \
+ || (__GLIBC__ > 2))))
+
+UTIL_time_t UTIL_getTime(void)
+{
+ UTIL_time_t time;
+ if (clock_gettime(CLOCK_MONOTONIC, &time))
+ UTIL_DISPLAYLEVEL(1, "ERROR: Failed to get time\n"); /* we could also exit() */
+ return time;
+}
+
+UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end)
+{
+ UTIL_time_t diff;
+ if (end.tv_nsec < begin.tv_nsec) {
+ diff.tv_sec = (end.tv_sec - 1) - begin.tv_sec;
+ diff.tv_nsec = (end.tv_nsec + 1000000000ULL) - begin.tv_nsec;
+ } else {
+ diff.tv_sec = end.tv_sec - begin.tv_sec;
+ diff.tv_nsec = end.tv_nsec - begin.tv_nsec;
+ }
+ return diff;
+}
+
+U64 UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end)
+{
+ UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
+ U64 micro = 0;
+ micro += 1000000ULL * diff.tv_sec;
+ micro += diff.tv_nsec / 1000ULL;
+ return micro;
+}
+
+U64 UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end)
+{
+ UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
+ U64 nano = 0;
+ nano += 1000000000ULL * diff.tv_sec;
+ nano += diff.tv_nsec;
+ return nano;
+}
+
+#else /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */
+
+UTIL_time_t UTIL_getTime(void) { return clock(); }
+U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
+U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
+
+#endif
+
+/* returns time span in microseconds */
+U64 UTIL_clockSpanMicro(UTIL_time_t clockStart )
+{
+ UTIL_time_t const clockEnd = UTIL_getTime();
+ return UTIL_getSpanTimeMicro(clockStart, clockEnd);
+}
+
+/* returns time span in microseconds */
+U64 UTIL_clockSpanNano(UTIL_time_t clockStart )
+{
+ UTIL_time_t const clockEnd = UTIL_getTime();
+ return UTIL_getSpanTimeNano(clockStart, clockEnd);
+}
+
+void UTIL_waitForNextTick(void)
+{
+ UTIL_time_t const clockStart = UTIL_getTime();
+ UTIL_time_t clockEnd;
+ do {
+ clockEnd = UTIL_getTime();
+ } while (UTIL_getSpanTimeNano(clockStart, clockEnd) == 0);
+}
+
+/* count the number of physical cores */
+#if defined(_WIN32) || defined(WIN32)
+
+#include <windows.h>
+
+typedef BOOL(WINAPI* LPFN_GLPI)(PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD);
+
+int UTIL_countPhysicalCores(void)
+{
+ static int numPhysicalCores = 0;
+ if (numPhysicalCores != 0) return numPhysicalCores;
+
+ { LPFN_GLPI glpi;
+ BOOL done = FALSE;
+ PSYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer = NULL;
+ PSYSTEM_LOGICAL_PROCESSOR_INFORMATION ptr = NULL;
+ DWORD returnLength = 0;
+ size_t byteOffset = 0;
+
+ glpi = (LPFN_GLPI)GetProcAddress(GetModuleHandle(TEXT("kernel32")),
+ "GetLogicalProcessorInformation");
+
+ if (glpi == NULL) {
+ goto failed;
+ }
+
+ while(!done) {
+ DWORD rc = glpi(buffer, &returnLength);
+ if (FALSE == rc) {
+ if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) {
+ if (buffer)
+ free(buffer);
+ buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)malloc(returnLength);
+
+ if (buffer == NULL) {
+ perror("zstd");
+ exit(1);
+ }
+ } else {
+ /* some other error */
+ goto failed;
+ }
+ } else {
+ done = TRUE;
+ }
+ }
+
+ ptr = buffer;
+
+ while (byteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= returnLength) {
+
+ if (ptr->Relationship == RelationProcessorCore) {
+ numPhysicalCores++;
+ }
+
+ ptr++;
+ byteOffset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
+ }
+
+ free(buffer);
+
+ return numPhysicalCores;
+ }
+
+failed:
+ /* try to fall back on GetSystemInfo */
+ { SYSTEM_INFO sysinfo;
+ GetSystemInfo(&sysinfo);
+ numPhysicalCores = sysinfo.dwNumberOfProcessors;
+ if (numPhysicalCores == 0) numPhysicalCores = 1; /* just in case */
+ }
+ return numPhysicalCores;
+}
+
+#elif defined(__APPLE__)
+
+#include <sys/sysctl.h>
+
+/* Use apple-provided syscall
+ * see: man 3 sysctl */
+int UTIL_countPhysicalCores(void)
+{
+ static S32 numPhysicalCores = 0; /* apple specifies int32_t */
+ if (numPhysicalCores != 0) return numPhysicalCores;
+
+ { size_t size = sizeof(S32);
+ int const ret = sysctlbyname("hw.physicalcpu", &numPhysicalCores, &size, NULL, 0);
+ if (ret != 0) {
+ if (errno == ENOENT) {
+ /* entry not present, fall back on 1 */
+ numPhysicalCores = 1;
+ } else {
+ perror("zstd: can't get number of physical cpus");
+ exit(1);
+ }
+ }
+
+ return numPhysicalCores;
+ }
+}
+
+#elif defined(__linux__)
+
+/* parse /proc/cpuinfo
+ * siblings / cpu cores should give hyperthreading ratio
+ * otherwise fall back on sysconf */
+int UTIL_countPhysicalCores(void)
+{
+ static int numPhysicalCores = 0;
+
+ if (numPhysicalCores != 0) return numPhysicalCores;
+
+ numPhysicalCores = (int)sysconf(_SC_NPROCESSORS_ONLN);
+ if (numPhysicalCores == -1) {
+ /* value not queryable, fall back on 1 */
+ return numPhysicalCores = 1;
+ }
+
+ /* try to determine if there's hyperthreading */
+ { FILE* const cpuinfo = fopen("/proc/cpuinfo", "r");
+#define BUF_SIZE 80
+ char buff[BUF_SIZE];
+
+ int siblings = 0;
+ int cpu_cores = 0;
+ int ratio = 1;
+
+ if (cpuinfo == NULL) {
+ /* fall back on the sysconf value */
+ return numPhysicalCores;
+ }
+
+ /* assume the cpu cores/siblings values will be constant across all
+ * present processors */
+ while (!feof(cpuinfo)) {
+ if (fgets(buff, BUF_SIZE, cpuinfo) != NULL) {
+ if (strncmp(buff, "siblings", 8) == 0) {
+ const char* const sep = strchr(buff, ':');
+ if (*sep == '\0') {
+ /* formatting was broken? */
+ goto failed;
+ }
+
+ siblings = atoi(sep + 1);
+ }
+ if (strncmp(buff, "cpu cores", 9) == 0) {
+ const char* const sep = strchr(buff, ':');
+ if (*sep == '\0') {
+ /* formatting was broken? */
+ goto failed;
+ }
+
+ cpu_cores = atoi(sep + 1);
+ }
+ } else if (ferror(cpuinfo)) {
+ /* fall back on the sysconf value */
+ goto failed;
+ }
+ }
+ if (siblings && cpu_cores) {
+ ratio = siblings / cpu_cores;
+ }
+failed:
+ fclose(cpuinfo);
+ return numPhysicalCores = numPhysicalCores / ratio;
+ }
+}
+
+#elif defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)
+
+/* Use apple-provided syscall
+ * see: man 3 sysctl */
+int UTIL_countPhysicalCores(void)
+{
+ static int numPhysicalCores = 0;
+
+ if (numPhysicalCores != 0) return numPhysicalCores;
+
+ numPhysicalCores = (int)sysconf(_SC_NPROCESSORS_ONLN);
+ if (numPhysicalCores == -1) {
+ /* value not queryable, fall back on 1 */
+ return numPhysicalCores = 1;
+ }
+ return numPhysicalCores;
+}
+
+#else
+
+int UTIL_countPhysicalCores(void)
+{
+ /* assume 1 */
+ return 1;
+}
+
+#endif
+
+#if defined (__cplusplus)
+}
+#endif
diff --git a/programs/util.h b/programs/util.h
index 67aa7a56b967c..f78bcbe1b3f8c 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -16,15 +16,13 @@ extern "C" {
#endif
-
/*-****************************************
* Dependencies
******************************************/
#include "platform.h" /* PLATFORM_POSIX_VERSION, ZSTD_NANOSLEEP_SUPPORT, ZSTD_SETPRIORITY_SUPPORT */
-#include <stdlib.h> /* malloc */
+#include <stdlib.h> /* malloc, realloc, free */
#include <stddef.h> /* size_t, ptrdiff_t */
#include <stdio.h> /* fprintf */
-#include <string.h> /* strncmp */
#include <sys/types.h> /* stat, utime */
#include <sys/stat.h> /* stat, chmod */
#if defined(_MSC_VER)
@@ -35,11 +33,10 @@ extern "C" {
# include <utime.h> /* utime */
#endif
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC, nanosleep */
-#include <errno.h>
#include "mem.h" /* U32, U64 */
-/* ************************************************************
+/*-************************************************************
* Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW
***************************************************************/
#if defined(_MSC_VER) && (_MSC_VER >= 1400)
@@ -84,7 +81,7 @@ extern "C" {
#endif
-/* *************************************
+/*-*************************************
* Constants
***************************************/
#define LIST_SIZE_INCREASE (8*1024)
@@ -110,7 +107,7 @@ extern "C" {
/*-****************************************
* Console log
******************************************/
-static int g_utilDisplayLevel;
+extern int g_utilDisplayLevel;
#define UTIL_DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define UTIL_DISPLAYLEVEL(l, ...) { if (g_utilDisplayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } }
@@ -119,61 +116,16 @@ static int g_utilDisplayLevel;
* Time functions
******************************************/
#if defined(_WIN32) /* Windows */
+
#define UTIL_TIME_INITIALIZER { { 0, 0 } }
typedef LARGE_INTEGER UTIL_time_t;
- UTIL_STATIC UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; }
- UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
- {
- static LARGE_INTEGER ticksPerSecond;
- static int init = 0;
- if (!init) {
- if (!QueryPerformanceFrequency(&ticksPerSecond))
- UTIL_DISPLAYLEVEL(1, "ERROR: QueryPerformanceFrequency() failure\n");
- init = 1;
- }
- return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
- }
- UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
- {
- static LARGE_INTEGER ticksPerSecond;
- static int init = 0;
- if (!init) {
- if (!QueryPerformanceFrequency(&ticksPerSecond))
- UTIL_DISPLAYLEVEL(1, "ERROR: QueryPerformanceFrequency() failure\n");
- init = 1;
- }
- return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
- }
-
#elif defined(__APPLE__) && defined(__MACH__)
#include <mach/mach_time.h>
#define UTIL_TIME_INITIALIZER 0
typedef U64 UTIL_time_t;
- UTIL_STATIC UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); }
- UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
- {
- static mach_timebase_info_data_t rate;
- static int init = 0;
- if (!init) {
- mach_timebase_info(&rate);
- init = 1;
- }
- return (((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom))/1000ULL;
- }
- UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
- {
- static mach_timebase_info_data_t rate;
- static int init = 0;
- if (!init) {
- mach_timebase_info(&rate);
- init = 1;
- }
- return ((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom);
- }
-
#elif (PLATFORM_POSIX_VERSION >= 200112L) \
&& (defined(__UCLIBC__) \
|| (defined(__GLIBC__) \
@@ -184,79 +136,27 @@ static int g_utilDisplayLevel;
typedef struct timespec UTIL_freq_t;
typedef struct timespec UTIL_time_t;
- UTIL_STATIC UTIL_time_t UTIL_getTime(void)
- {
- UTIL_time_t time;
- if (clock_gettime(CLOCK_MONOTONIC, &time))
- UTIL_DISPLAYLEVEL(1, "ERROR: Failed to get time\n"); /* we could also exit() */
- return time;
- }
-
- UTIL_STATIC UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end)
- {
- UTIL_time_t diff;
- if (end.tv_nsec < begin.tv_nsec) {
- diff.tv_sec = (end.tv_sec - 1) - begin.tv_sec;
- diff.tv_nsec = (end.tv_nsec + 1000000000ULL) - begin.tv_nsec;
- } else {
- diff.tv_sec = end.tv_sec - begin.tv_sec;
- diff.tv_nsec = end.tv_nsec - begin.tv_nsec;
- }
- return diff;
- }
-
- UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end)
- {
- UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
- U64 micro = 0;
- micro += 1000000ULL * diff.tv_sec;
- micro += diff.tv_nsec / 1000ULL;
- return micro;
- }
-
- UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end)
- {
- UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
- U64 nano = 0;
- nano += 1000000000ULL * diff.tv_sec;
- nano += diff.tv_nsec;
- return nano;
- }
+ UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end);
#else /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */
+
typedef clock_t UTIL_time_t;
#define UTIL_TIME_INITIALIZER 0
- UTIL_STATIC UTIL_time_t UTIL_getTime(void) { return clock(); }
- UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
- UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
+
#endif
+UTIL_time_t UTIL_getTime(void);
+U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd);
+U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd);
+
#define SEC_TO_MICRO 1000000
/* returns time span in microseconds */
-UTIL_STATIC U64 UTIL_clockSpanMicro(UTIL_time_t clockStart )
-{
- UTIL_time_t const clockEnd = UTIL_getTime();
- return UTIL_getSpanTimeMicro(clockStart, clockEnd);
-}
+U64 UTIL_clockSpanMicro(UTIL_time_t clockStart);
/* returns time span in microseconds */
-UTIL_STATIC U64 UTIL_clockSpanNano(UTIL_time_t clockStart )
-{
- UTIL_time_t const clockEnd = UTIL_getTime();
- return UTIL_getSpanTimeNano(clockStart, clockEnd);
-}
-
-UTIL_STATIC void UTIL_waitForNextTick(void)
-{
- UTIL_time_t const clockStart = UTIL_getTime();
- UTIL_time_t clockEnd;
- do {
- clockEnd = UTIL_getTime();
- } while (UTIL_getSpanTimeNano(clockStart, clockEnd) == 0);
-}
-
-
+U64 UTIL_clockSpanNano(UTIL_time_t clockStart);
+void UTIL_waitForNextTick(void);
/*-****************************************
* File functions
@@ -269,129 +169,23 @@ UTIL_STATIC void UTIL_waitForNextTick(void)
#endif
-UTIL_STATIC int UTIL_isRegularFile(const char* infilename);
-
-
-UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf)
-{
- int res = 0;
- struct utimbuf timebuf;
-
- if (!UTIL_isRegularFile(filename))
- return -1;
-
- timebuf.actime = time(NULL);
- timebuf.modtime = statbuf->st_mtime;
- res += utime(filename, &timebuf); /* set access and modification times */
-
-#if !defined(_WIN32)
- res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */
-#endif
-
- res += chmod(filename, statbuf->st_mode & 07777); /* Copy file permissions */
-
- errno = 0;
- return -res; /* number of errors is returned */
-}
-
-
-UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf)
-{
- int r;
-#if defined(_MSC_VER)
- r = _stat64(infilename, statbuf);
- if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */
-#else
- r = stat(infilename, statbuf);
- if (r || !S_ISREG(statbuf->st_mode)) return 0; /* No good... */
-#endif
- return 1;
-}
-
-
-UTIL_STATIC int UTIL_isRegularFile(const char* infilename)
-{
- stat_t statbuf;
- return UTIL_getFileStat(infilename, &statbuf); /* Only need to know whether it is a regular file */
-}
-
-
-UTIL_STATIC U32 UTIL_isDirectory(const char* infilename)
-{
- int r;
- stat_t statbuf;
-#if defined(_MSC_VER)
- r = _stat64(infilename, &statbuf);
- if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
-#else
- r = stat(infilename, &statbuf);
- if (!r && S_ISDIR(statbuf.st_mode)) return 1;
-#endif
- return 0;
-}
-
-UTIL_STATIC U32 UTIL_isLink(const char* infilename)
-{
-/* macro guards, as defined in : https://linux.die.net/man/2/lstat */
-#ifndef __STRICT_ANSI__
-#if defined(_BSD_SOURCE) \
- || (defined(_XOPEN_SOURCE) && (_XOPEN_SOURCE >= 500)) \
- || (defined(_XOPEN_SOURCE) && defined(_XOPEN_SOURCE_EXTENDED)) \
- || (defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) \
- || (defined(__APPLE__) && defined(__MACH__))
- int r;
- stat_t statbuf;
- r = lstat(infilename, &statbuf);
- if (!r && S_ISLNK(statbuf.st_mode)) return 1;
-#endif
-#endif
- (void)infilename;
- return 0;
-}
-
+int UTIL_fileExist(const char* filename);
+int UTIL_isRegularFile(const char* infilename);
+int UTIL_setFileStat(const char* filename, stat_t* statbuf);
+U32 UTIL_isDirectory(const char* infilename);
+int UTIL_getFileStat(const char* infilename, stat_t* statbuf);
+U32 UTIL_isLink(const char* infilename);
#define UTIL_FILESIZE_UNKNOWN ((U64)(-1))
-UTIL_STATIC U64 UTIL_getFileSize(const char* infilename)
-{
- if (!UTIL_isRegularFile(infilename)) return UTIL_FILESIZE_UNKNOWN;
- { int r;
-#if defined(_MSC_VER)
- struct __stat64 statbuf;
- r = _stat64(infilename, &statbuf);
- if (r || !(statbuf.st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN;
-#elif defined(__MINGW32__) && defined (__MSVCRT__)
- struct _stati64 statbuf;
- r = _stati64(infilename, &statbuf);
- if (r || !(statbuf.st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN;
-#else
- struct stat statbuf;
- r = stat(infilename, &statbuf);
- if (r || !S_ISREG(statbuf.st_mode)) return UTIL_FILESIZE_UNKNOWN;
-#endif
- return (U64)statbuf.st_size;
- }
-}
-
-
-UTIL_STATIC U64 UTIL_getTotalFileSize(const char* const * const fileNamesTable, unsigned nbFiles)
-{
- U64 total = 0;
- int error = 0;
- unsigned n;
- for (n=0; n<nbFiles; n++) {
- U64 const size = UTIL_getFileSize(fileNamesTable[n]);
- error |= (size == UTIL_FILESIZE_UNKNOWN);
- total += size;
- }
- return error ? UTIL_FILESIZE_UNKNOWN : total;
-}
+U64 UTIL_getFileSize(const char* infilename);
+U64 UTIL_getTotalFileSize(const char* const * const fileNamesTable, unsigned nbFiles);
/*
* A modified version of realloc().
* If UTIL_realloc() fails the original block is freed.
*/
-UTIL_STATIC void *UTIL_realloc(void *ptr, size_t size)
+UTIL_STATIC void* UTIL_realloc(void *ptr, size_t size)
{
void *newptr = realloc(ptr, size);
if (newptr) return newptr;
@@ -399,143 +193,14 @@ UTIL_STATIC void *UTIL_realloc(void *ptr, size_t size)
return NULL;
}
+int UTIL_prepareFileList(const char* dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks);
#ifdef _WIN32
# define UTIL_HAS_CREATEFILELIST
-
-UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
-{
- char* path;
- int dirLength, fnameLength, pathLength, nbFiles = 0;
- WIN32_FIND_DATAA cFile;
- HANDLE hFile;
-
- dirLength = (int)strlen(dirName);
- path = (char*) malloc(dirLength + 3);
- if (!path) return 0;
-
- memcpy(path, dirName, dirLength);
- path[dirLength] = '\\';
- path[dirLength+1] = '*';
- path[dirLength+2] = 0;
-
- hFile=FindFirstFileA(path, &cFile);
- if (hFile == INVALID_HANDLE_VALUE) {
- UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s'\n", dirName);
- return 0;
- }
- free(path);
-
- do {
- fnameLength = (int)strlen(cFile.cFileName);
- path = (char*) malloc(dirLength + fnameLength + 2);
- if (!path) { FindClose(hFile); return 0; }
- memcpy(path, dirName, dirLength);
- path[dirLength] = '\\';
- memcpy(path+dirLength+1, cFile.cFileName, fnameLength);
- pathLength = dirLength+1+fnameLength;
- path[pathLength] = 0;
- if (cFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
- if (strcmp (cFile.cFileName, "..") == 0 ||
- strcmp (cFile.cFileName, ".") == 0) continue;
-
- nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */
- if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
- }
- else if ((cFile.dwFileAttributes & FILE_ATTRIBUTE_NORMAL) || (cFile.dwFileAttributes & FILE_ATTRIBUTE_ARCHIVE) || (cFile.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED)) {
- if (*bufStart + *pos + pathLength >= *bufEnd) {
- ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
- *bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
- *bufEnd = *bufStart + newListSize;
- if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
- }
- if (*bufStart + *pos + pathLength < *bufEnd) {
- strncpy(*bufStart + *pos, path, *bufEnd - (*bufStart + *pos));
- *pos += pathLength + 1;
- nbFiles++;
- }
- }
- free(path);
- } while (FindNextFileA(hFile, &cFile));
-
- FindClose(hFile);
- return nbFiles;
-}
-
#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */
# define UTIL_HAS_CREATEFILELIST
# include <dirent.h> /* opendir, readdir */
# include <string.h> /* strerror, memcpy */
-
-UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
-{
- DIR *dir;
- struct dirent *entry;
- char* path;
- int dirLength, fnameLength, pathLength, nbFiles = 0;
-
- if (!(dir = opendir(dirName))) {
- UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno));
- return 0;
- }
-
- dirLength = (int)strlen(dirName);
- errno = 0;
- while ((entry = readdir(dir)) != NULL) {
- if (strcmp (entry->d_name, "..") == 0 ||
- strcmp (entry->d_name, ".") == 0) continue;
- fnameLength = (int)strlen(entry->d_name);
- path = (char*) malloc(dirLength + fnameLength + 2);
- if (!path) { closedir(dir); return 0; }
- memcpy(path, dirName, dirLength);
-
- path[dirLength] = '/';
- memcpy(path+dirLength+1, entry->d_name, fnameLength);
- pathLength = dirLength+1+fnameLength;
- path[pathLength] = 0;
-
- if (!followLinks && UTIL_isLink(path)) {
- UTIL_DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", path);
- continue;
- }
-
- if (UTIL_isDirectory(path)) {
- nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */
- if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
- } else {
- if (*bufStart + *pos + pathLength >= *bufEnd) {
- ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
- *bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
- *bufEnd = *bufStart + newListSize;
- if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
- }
- if (*bufStart + *pos + pathLength < *bufEnd) {
- strncpy(*bufStart + *pos, path, *bufEnd - (*bufStart + *pos));
- *pos += pathLength + 1;
- nbFiles++;
- }
- }
- free(path);
- errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */
- }
-
- if (errno != 0) {
- UTIL_DISPLAYLEVEL(1, "readdir(%s) error: %s\n", dirName, strerror(errno));
- free(*bufStart);
- *bufStart = NULL;
- }
- closedir(dir);
- return nbFiles;
-}
-
#else
-
-UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
-{
- (void)bufStart; (void)bufEnd; (void)pos; (void)followLinks;
- UTIL_DISPLAYLEVEL(1, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName);
- return 0;
-}
-
#endif /* #ifdef _WIN32 */
/*
@@ -544,56 +209,10 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
* After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer)
* In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called.
*/
-UTIL_STATIC const char**
+const char**
UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
char** allocatedBuffer, unsigned* allocatedNamesNb,
- int followLinks)
-{
- size_t pos;
- unsigned i, nbFiles;
- char* buf = (char*)malloc(LIST_SIZE_INCREASE);
- char* bufend = buf + LIST_SIZE_INCREASE;
- const char** fileTable;
-
- if (!buf) return NULL;
-
- for (i=0, pos=0, nbFiles=0; i<inputNamesNb; i++) {
- if (!UTIL_isDirectory(inputNames[i])) {
- size_t const len = strlen(inputNames[i]);
- if (buf + pos + len >= bufend) {
- ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
- buf = (char*)UTIL_realloc(buf, newListSize);
- bufend = buf + newListSize;
- if (!buf) return NULL;
- }
- if (buf + pos + len < bufend) {
- strncpy(buf + pos, inputNames[i], bufend - (buf + pos));
- pos += len + 1;
- nbFiles++;
- }
- } else {
- nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks);
- if (buf == NULL) return NULL;
- } }
-
- if (nbFiles == 0) { free(buf); return NULL; }
-
- fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*));
- if (!fileTable) { free(buf); return NULL; }
-
- for (i=0, pos=0; i<nbFiles; i++) {
- fileTable[i] = buf + pos;
- pos += strlen(fileTable[i]) + 1;
- }
-
- if (buf + pos > bufend) { free(buf); free((void*)fileTable); return NULL; }
-
- *allocatedBuffer = buf;
- *allocatedNamesNb = nbFiles;
-
- return fileTable;
-}
-
+ int followLinks);
UTIL_STATIC void UTIL_freeFileList(const char** filenameTable, char* allocatedBuffer)
{
@@ -601,201 +220,7 @@ UTIL_STATIC void UTIL_freeFileList(const char** filenameTable, char* allocatedBu
if (filenameTable) free((void*)filenameTable);
}
-/* count the number of physical cores */
-#if defined(_WIN32) || defined(WIN32)
-
-#include <windows.h>
-
-typedef BOOL(WINAPI* LPFN_GLPI)(PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD);
-
-UTIL_STATIC int UTIL_countPhysicalCores(void)
-{
- static int numPhysicalCores = 0;
- if (numPhysicalCores != 0) return numPhysicalCores;
-
- { LPFN_GLPI glpi;
- BOOL done = FALSE;
- PSYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer = NULL;
- PSYSTEM_LOGICAL_PROCESSOR_INFORMATION ptr = NULL;
- DWORD returnLength = 0;
- size_t byteOffset = 0;
-
- glpi = (LPFN_GLPI)GetProcAddress(GetModuleHandle(TEXT("kernel32")),
- "GetLogicalProcessorInformation");
-
- if (glpi == NULL) {
- goto failed;
- }
-
- while(!done) {
- DWORD rc = glpi(buffer, &returnLength);
- if (FALSE == rc) {
- if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) {
- if (buffer)
- free(buffer);
- buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)malloc(returnLength);
-
- if (buffer == NULL) {
- perror("zstd");
- exit(1);
- }
- } else {
- /* some other error */
- goto failed;
- }
- } else {
- done = TRUE;
- }
- }
-
- ptr = buffer;
-
- while (byteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= returnLength) {
-
- if (ptr->Relationship == RelationProcessorCore) {
- numPhysicalCores++;
- }
-
- ptr++;
- byteOffset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
- }
-
- free(buffer);
-
- return numPhysicalCores;
- }
-
-failed:
- /* try to fall back on GetSystemInfo */
- { SYSTEM_INFO sysinfo;
- GetSystemInfo(&sysinfo);
- numPhysicalCores = sysinfo.dwNumberOfProcessors;
- if (numPhysicalCores == 0) numPhysicalCores = 1; /* just in case */
- }
- return numPhysicalCores;
-}
-
-#elif defined(__APPLE__)
-
-#include <sys/sysctl.h>
-
-/* Use apple-provided syscall
- * see: man 3 sysctl */
-UTIL_STATIC int UTIL_countPhysicalCores(void)
-{
- static S32 numPhysicalCores = 0; /* apple specifies int32_t */
- if (numPhysicalCores != 0) return numPhysicalCores;
-
- { size_t size = sizeof(S32);
- int const ret = sysctlbyname("hw.physicalcpu", &numPhysicalCores, &size, NULL, 0);
- if (ret != 0) {
- if (errno == ENOENT) {
- /* entry not present, fall back on 1 */
- numPhysicalCores = 1;
- } else {
- perror("zstd: can't get number of physical cpus");
- exit(1);
- }
- }
-
- return numPhysicalCores;
- }
-}
-
-#elif defined(__linux__)
-
-/* parse /proc/cpuinfo
- * siblings / cpu cores should give hyperthreading ratio
- * otherwise fall back on sysconf */
-UTIL_STATIC int UTIL_countPhysicalCores(void)
-{
- static int numPhysicalCores = 0;
-
- if (numPhysicalCores != 0) return numPhysicalCores;
-
- numPhysicalCores = (int)sysconf(_SC_NPROCESSORS_ONLN);
- if (numPhysicalCores == -1) {
- /* value not queryable, fall back on 1 */
- return numPhysicalCores = 1;
- }
-
- /* try to determine if there's hyperthreading */
- { FILE* const cpuinfo = fopen("/proc/cpuinfo", "r");
-#define BUF_SIZE 80
- char buff[BUF_SIZE];
-
- int siblings = 0;
- int cpu_cores = 0;
- int ratio = 1;
-
- if (cpuinfo == NULL) {
- /* fall back on the sysconf value */
- return numPhysicalCores;
- }
-
- /* assume the cpu cores/siblings values will be constant across all
- * present processors */
- while (!feof(cpuinfo)) {
- if (fgets(buff, BUF_SIZE, cpuinfo) != NULL) {
- if (strncmp(buff, "siblings", 8) == 0) {
- const char* const sep = strchr(buff, ':');
- if (*sep == '\0') {
- /* formatting was broken? */
- goto failed;
- }
-
- siblings = atoi(sep + 1);
- }
- if (strncmp(buff, "cpu cores", 9) == 0) {
- const char* const sep = strchr(buff, ':');
- if (*sep == '\0') {
- /* formatting was broken? */
- goto failed;
- }
-
- cpu_cores = atoi(sep + 1);
- }
- } else if (ferror(cpuinfo)) {
- /* fall back on the sysconf value */
- goto failed;
- }
- }
- if (siblings && cpu_cores) {
- ratio = siblings / cpu_cores;
- }
-failed:
- fclose(cpuinfo);
- return numPhysicalCores = numPhysicalCores / ratio;
- }
-}
-
-#elif defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)
-
-/* Use apple-provided syscall
- * see: man 3 sysctl */
-UTIL_STATIC int UTIL_countPhysicalCores(void)
-{
- static int numPhysicalCores = 0;
-
- if (numPhysicalCores != 0) return numPhysicalCores;
-
- numPhysicalCores = (int)sysconf(_SC_NPROCESSORS_ONLN);
- if (numPhysicalCores == -1) {
- /* value not queryable, fall back on 1 */
- return numPhysicalCores = 1;
- }
- return numPhysicalCores;
-}
-
-#else
-
-UTIL_STATIC int UTIL_countPhysicalCores(void)
-{
- /* assume 1 */
- return 1;
-}
-
-#endif
+int UTIL_countPhysicalCores(void);
#if defined (__cplusplus)
}
diff --git a/programs/zstd.1 b/programs/zstd.1
index 674f89841ce8a..c93755f877edd 100644
--- a/programs/zstd.1
+++ b/programs/zstd.1
@@ -1,5 +1,5 @@
.
-.TH "ZSTD" "1" "October 2018" "zstd 1.3.7" "User Commands"
+.TH "ZSTD" "1" "December 2018" "zstd 1.3.8" "User Commands"
.
.SH "NAME"
\fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
@@ -127,6 +127,10 @@ Does not spawn a thread for compression, use a single thread for both I/O and co
\fBzstd\fR will dynamically adapt compression level to perceived I/O conditions\. Compression level adaptation can be observed live by using command \fB\-v\fR\. Adaptation can be constrained between supplied \fBmin\fR and \fBmax\fR levels\. The feature works when combined with multi\-threading and \fB\-\-long\fR mode\. It does not work with \fB\-\-single\-thread\fR\. It sets window size to 8 MB by default (can be changed manually, see \fBwlog\fR)\. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible\. \fInote\fR : at the time of this writing, \fB\-\-adapt\fR can remain stuck at low speed when combined with multiple worker threads (>=2)\.
.
.TP
+\fB\-\-rsyncable\fR
+\fBzstd\fR will periodically synchronize the compression state to make the compressed file more rsync\-friendly\. There is a negligible impact to compression ratio, and the faster compression levels will see a small compression speed hit\. This feature does not work with \fB\-\-single\-thread\fR\. You probably don\'t want to use it with long range mode, since it will decrease the effectiveness of the synchronization points, but your milage may vary\.
+.
+.TP
\fB\-D file\fR
use \fBfile\fR as Dictionary to compress or decompress FILE(s)
.
@@ -312,7 +316,7 @@ set process priority to real\-time
Specify a strategy used by a match finder\.
.
.IP
-There are 8 strategies numbered from 1 to 8, from faster to stronger: 1=ZSTD_fast, 2=ZSTD_dfast, 3=ZSTD_greedy, 4=ZSTD_lazy, 5=ZSTD_lazy2, 6=ZSTD_btlazy2, 7=ZSTD_btopt, 8=ZSTD_btultra\.
+There are 9 strategies numbered from 1 to 9, from faster to stronger: 1=ZSTD_fast, 2=ZSTD_dfast, 3=ZSTD_greedy, 4=ZSTD_lazy, 5=ZSTD_lazy2, 6=ZSTD_btlazy2, 7=ZSTD_btopt, 8=ZSTD_btultra, 9=ZSTD_btultra2\.
.
.TP
\fBwindowLog\fR=\fIwlog\fR, \fBwlog\fR=\fIwlog\fR
@@ -355,21 +359,21 @@ More searches increases the chance to find a match which usually increases compr
The minimum \fIslog\fR is 1 and the maximum is 26\.
.
.TP
-\fBsearchLength\fR=\fIslen\fR, \fBslen\fR=\fIslen\fR
+\fBminMatch\fR=\fImml\fR, \fBmml\fR=\fImml\fR
Specify the minimum searched length of a match in a hash table\.
.
.IP
Larger search lengths usually decrease compression ratio but improve decompression speed\.
.
.IP
-The minimum \fIslen\fR is 3 and the maximum is 7\.
+The minimum \fImml\fR is 3 and the maximum is 7\.
.
.TP
\fBtargetLen\fR=\fItlen\fR, \fBtlen\fR=\fItlen\fR
The impact of this field vary depending on selected strategy\.
.
.IP
-For ZSTD_btopt and ZSTD_btultra, it specifies the minimum match length that causes match finder to stop searching for better matches\. A larger \fBtargetLen\fR usually improves compression ratio but decreases compression speed\.
+For ZSTD_btopt, ZSTD_btultra and ZSTD_btultra2, it specifies the minimum match length that causes match finder to stop searching\. A larger \fBtargetLen\fR usually improves compression ratio but decreases compression speed\.
.
.IP
For ZSTD_fast, it triggers ultra\-fast mode when > 0\. The value represents the amount of data skipped between match sampling\. Impact is reversed : a larger \fBtargetLen\fR increases compression speed but decreases compression ratio\.
@@ -385,10 +389,10 @@ The minimum \fItlen\fR is 0 and the maximum is 999\.
Determine \fBoverlapSize\fR, amount of data reloaded from previous job\. This parameter is only available when multithreading is enabled\. Reloading more data improves compression ratio, but decreases speed\.
.
.IP
-The minimum \fIovlog\fR is 0, and the maximum is 9\. 0 means "no overlap", hence completely independent jobs\. 9 means "full overlap", meaning up to \fBwindowSize\fR is reloaded from previous job\. Reducing \fIovlog\fR by 1 reduces the amount of reload by a factor 2\. Default \fIovlog\fR is 6, which means "reload \fBwindowSize / 8\fR"\. Exception : the maximum compression level (22) has a default \fIovlog\fR of 9\.
+The minimum \fIovlog\fR is 0, and the maximum is 9\. 1 means "no overlap", hence completely independent jobs\. 9 means "full overlap", meaning up to \fBwindowSize\fR is reloaded from previous job\. Reducing \fIovlog\fR by 1 reduces the reloaded amount by a factor 2\. For example, 8 means "windowSize/2", and 6 means "windowSize/8"\. Value 0 is special and means "default" : \fIovlog\fR is automatically determined by \fBzstd\fR\. In which case, \fIovlog\fR will range from 6 to 9, depending on selected \fIstrat\fR\.
.
.TP
-\fBldmHashLog\fR=\fIldmhlog\fR, \fBldmhlog\fR=\fIldmhlog\fR
+\fBldmHashLog\fR=\fIlhlog\fR, \fBlhlog\fR=\fIlhlog\fR
Specify the maximum size for a hash table used for long distance matching\.
.
.IP
@@ -398,10 +402,10 @@ This option is ignored unless long distance matching is enabled\.
Bigger hash tables usually improve compression ratio at the expense of more memory during compression and a decrease in compression speed\.
.
.IP
-The minimum \fIldmhlog\fR is 6 and the maximum is 26 (default: 20)\.
+The minimum \fIlhlog\fR is 6 and the maximum is 26 (default: 20)\.
.
.TP
-\fBldmSearchLength\fR=\fIldmslen\fR, \fBldmslen\fR=\fIldmslen\fR
+\fBldmMinMatch\fR=\fIlmml\fR, \fBlmml\fR=\fIlmml\fR
Specify the minimum searched length of a match for long distance matching\.
.
.IP
@@ -411,10 +415,10 @@ This option is ignored unless long distance matching is enabled\.
Larger/very small values usually decrease compression ratio\.
.
.IP
-The minimum \fIldmslen\fR is 4 and the maximum is 4096 (default: 64)\.
+The minimum \fIlmml\fR is 4 and the maximum is 4096 (default: 64)\.
.
.TP
-\fBldmBucketSizeLog\fR=\fIldmblog\fR, \fBldmblog\fR=\fIldmblog\fR
+\fBldmBucketSizeLog\fR=\fIlblog\fR, \fBlblog\fR=\fIlblog\fR
Specify the size of each bucket for the hash table used for long distance matching\.
.
.IP
@@ -424,10 +428,10 @@ This option is ignored unless long distance matching is enabled\.
Larger bucket sizes improve collision resolution but decrease compression speed\.
.
.IP
-The minimum \fIldmblog\fR is 0 and the maximum is 8 (default: 3)\.
+The minimum \fIlblog\fR is 0 and the maximum is 8 (default: 3)\.
.
.TP
-\fBldmHashEveryLog\fR=\fIldmhevery\fR, \fBldmhevery\fR=\fIldmhevery\fR
+\fBldmHashRateLog\fR=\fIlhrlog\fR, \fBlhrlog\fR=\fIlhrlog\fR
Specify the frequency of inserting entries into the long distance matching hash table\.
.
.IP
@@ -437,13 +441,13 @@ This option is ignored unless long distance matching is enabled\.
Larger values will improve compression speed\. Deviating far from the default value will likely result in a decrease in compression ratio\.
.
.IP
-The default value is \fBwlog \- ldmhlog\fR\.
+The default value is \fBwlog \- lhlog\fR\.
.
.SS "Example"
The following parameters sets advanced compression options to something similar to predefined level 19 for files bigger than 256 KB:
.
.P
-\fB\-\-zstd\fR=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6
+\fB\-\-zstd\fR=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6
.
.SS "\-B#:"
Select the size of each compression job\. This parameter is available only when multi\-threading is enabled\. Default value is \fB4 * windowSize\fR, which means it varies depending on compression level\. \fB\-B#\fR makes it possible to select a custom value\. Note that job size must respect a minimum value which is enforced transparently\. This minimum is either 1 MB, or \fBoverlapSize\fR, whichever is largest\.
diff --git a/programs/zstd.1.md b/programs/zstd.1.md
index c0c04698ddc2c..a029af5ff3cef 100644
--- a/programs/zstd.1.md
+++ b/programs/zstd.1.md
@@ -144,6 +144,14 @@ the last one takes effect.
Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible.
_note_ : at the time of this writing, `--adapt` can remain stuck at low speed
when combined with multiple worker threads (>=2).
+* `--rsyncable` :
+ `zstd` will periodically synchronize the compression state to make the
+ compressed file more rsync-friendly. There is a negligible impact to
+ compression ratio, and the faster compression levels will see a small
+ compression speed hit.
+ This feature does not work with `--single-thread`. You probably don't want
+ to use it with long range mode, since it will decrease the effectiveness of
+ the synchronization points, but your milage may vary.
* `-D file`:
use `file` as Dictionary to compress or decompress FILE(s)
* `--no-dictID`:
@@ -187,6 +195,8 @@ the last one takes effect.
* `-q`, `--quiet`:
suppress warnings, interactivity, and notifications.
specify twice to suppress errors too.
+* `--no-progress`:
+ do not display the progress bar, but keep all other messages.
* `-C`, `--[no-]check`:
add integrity check computed from uncompressed data (default: enabled)
* `--`:
@@ -331,9 +341,10 @@ The list of available _options_:
- `strategy`=_strat_, `strat`=_strat_:
Specify a strategy used by a match finder.
- There are 8 strategies numbered from 1 to 8, from faster to stronger:
- 1=ZSTD\_fast, 2=ZSTD\_dfast, 3=ZSTD\_greedy, 4=ZSTD\_lazy,
- 5=ZSTD\_lazy2, 6=ZSTD\_btlazy2, 7=ZSTD\_btopt, 8=ZSTD\_btultra.
+ There are 9 strategies numbered from 1 to 9, from faster to stronger:
+ 1=ZSTD\_fast, 2=ZSTD\_dfast, 3=ZSTD\_greedy,
+ 4=ZSTD\_lazy, 5=ZSTD\_lazy2, 6=ZSTD\_btlazy2,
+ 7=ZSTD\_btopt, 8=ZSTD\_btultra, 9=ZSTD\_btultra2.
- `windowLog`=_wlog_, `wlog`=_wlog_:
Specify the maximum number of bits for a match distance.
@@ -375,19 +386,19 @@ The list of available _options_:
The minimum _slog_ is 1 and the maximum is 26.
-- `searchLength`=_slen_, `slen`=_slen_:
+- `minMatch`=_mml_, `mml`=_mml_:
Specify the minimum searched length of a match in a hash table.
Larger search lengths usually decrease compression ratio but improve
decompression speed.
- The minimum _slen_ is 3 and the maximum is 7.
+ The minimum _mml_ is 3 and the maximum is 7.
- `targetLen`=_tlen_, `tlen`=_tlen_:
The impact of this field vary depending on selected strategy.
- For ZSTD\_btopt and ZSTD\_btultra, it specifies the minimum match length
- that causes match finder to stop searching for better matches.
+ For ZSTD\_btopt, ZSTD\_btultra and ZSTD\_btultra2, it specifies
+ the minimum match length that causes match finder to stop searching.
A larger `targetLen` usually improves compression ratio
but decreases compression speed.
@@ -406,13 +417,14 @@ The list of available _options_:
Reloading more data improves compression ratio, but decreases speed.
The minimum _ovlog_ is 0, and the maximum is 9.
- 0 means "no overlap", hence completely independent jobs.
+ 1 means "no overlap", hence completely independent jobs.
9 means "full overlap", meaning up to `windowSize` is reloaded from previous job.
- Reducing _ovlog_ by 1 reduces the amount of reload by a factor 2.
- Default _ovlog_ is 6, which means "reload `windowSize / 8`".
- Exception : the maximum compression level (22) has a default _ovlog_ of 9.
+ Reducing _ovlog_ by 1 reduces the reloaded amount by a factor 2.
+ For example, 8 means "windowSize/2", and 6 means "windowSize/8".
+ Value 0 is special and means "default" : _ovlog_ is automatically determined by `zstd`.
+ In which case, _ovlog_ will range from 6 to 9, depending on selected _strat_.
-- `ldmHashLog`=_ldmhlog_, `ldmhlog`=_ldmhlog_:
+- `ldmHashLog`=_lhlog_, `lhlog`=_lhlog_:
Specify the maximum size for a hash table used for long distance matching.
This option is ignored unless long distance matching is enabled.
@@ -420,18 +432,18 @@ The list of available _options_:
Bigger hash tables usually improve compression ratio at the expense of more
memory during compression and a decrease in compression speed.
- The minimum _ldmhlog_ is 6 and the maximum is 26 (default: 20).
+ The minimum _lhlog_ is 6 and the maximum is 26 (default: 20).
-- `ldmSearchLength`=_ldmslen_, `ldmslen`=_ldmslen_:
+- `ldmMinMatch`=_lmml_, `lmml`=_lmml_:
Specify the minimum searched length of a match for long distance matching.
This option is ignored unless long distance matching is enabled.
Larger/very small values usually decrease compression ratio.
- The minimum _ldmslen_ is 4 and the maximum is 4096 (default: 64).
+ The minimum _lmml_ is 4 and the maximum is 4096 (default: 64).
-- `ldmBucketSizeLog`=_ldmblog_, `ldmblog`=_ldmblog_:
+- `ldmBucketSizeLog`=_lblog_, `lblog`=_lblog_:
Specify the size of each bucket for the hash table used for long distance
matching.
@@ -440,9 +452,9 @@ The list of available _options_:
Larger bucket sizes improve collision resolution but decrease compression
speed.
- The minimum _ldmblog_ is 0 and the maximum is 8 (default: 3).
+ The minimum _lblog_ is 0 and the maximum is 8 (default: 3).
-- `ldmHashEveryLog`=_ldmhevery_, `ldmhevery`=_ldmhevery_:
+- `ldmHashRateLog`=_lhrlog_, `lhrlog`=_lhrlog_:
Specify the frequency of inserting entries into the long distance matching
hash table.
@@ -451,13 +463,13 @@ The list of available _options_:
Larger values will improve compression speed. Deviating far from the
default value will likely result in a decrease in compression ratio.
- The default value is `wlog - ldmhlog`.
+ The default value is `wlog - lhlog`.
### Example
The following parameters sets advanced compression options to something
similar to predefined level 19 for files bigger than 256 KB:
-`--zstd`=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6
+`--zstd`=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6
### -B#:
Select the size of each compression job.
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 1545d1cac5790..64dad110c7af5 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -28,11 +28,12 @@
#include "platform.h" /* IS_CONSOLE, PLATFORM_POSIX_VERSION */
#include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */
#include <stdio.h> /* fprintf(), stdin, stdout, stderr */
+#include <stdlib.h> /* getenv */
#include <string.h> /* strcmp, strlen */
#include <errno.h> /* errno */
#include "fileio.h" /* stdinmark, stdoutmark, ZSTD_EXTENSION */
#ifndef ZSTD_NOBENCH
-# include "bench.h" /* BMK_benchFiles */
+# include "benchzstd.h" /* BMK_benchFiles */
#endif
#ifndef ZSTD_NODICT
# include "dibio.h" /* ZDICT_cover_params_t, DiB_trainFromFiles() */
@@ -81,7 +82,7 @@ static const unsigned g_defaultMaxWindowLog = 27;
static U32 g_overlapLog = OVERLAP_LOG_DEFAULT;
static U32 g_ldmHashLog = 0;
static U32 g_ldmMinMatch = 0;
-static U32 g_ldmHashEveryLog = LDM_PARAM_DEFAULT;
+static U32 g_ldmHashRateLog = LDM_PARAM_DEFAULT;
static U32 g_ldmBucketSizeLog = LDM_PARAM_DEFAULT;
@@ -143,6 +144,7 @@ static int usage_advanced(const char* programName)
#ifdef ZSTD_MULTITHREAD
DISPLAY( " -T# : spawns # compression threads (default: 1, 0==# cores) \n");
DISPLAY( " -B# : select size of each job (default: 0==automatic) \n");
+ DISPLAY( " --rsyncable : compress using a rsync-friendly method (-B sets block size) \n");
#endif
DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n");
DISPLAY( "--[no-]check : integrity check (default: enabled) \n");
@@ -150,7 +152,7 @@ static int usage_advanced(const char* programName)
#ifdef UTIL_HAS_CREATEFILELIST
DISPLAY( " -r : operate recursively on directories \n");
#endif
- DISPLAY( "--format=zstd : compress files to the .zstd format (default) \n");
+ DISPLAY( "--format=zstd : compress files to the .zst format (default) \n");
#ifdef ZSTD_GZCOMPRESS
DISPLAY( "--format=gzip : compress files to the .gz format \n");
#endif
@@ -170,6 +172,7 @@ static int usage_advanced(const char* programName)
#endif
#endif
DISPLAY( " -M# : Set a memory usage limit for decompression \n");
+ DISPLAY( "--no-progress : do not display the progress bar \n");
DISPLAY( "-- : All arguments after \"--\" are treated as files \n");
#ifndef ZSTD_NODICT
DISPLAY( "\n");
@@ -231,32 +234,44 @@ static void errorOut(const char* msg)
DISPLAY("%s \n", msg); exit(1);
}
-/*! readU32FromChar() :
- * @return : unsigned integer value read from input in `char` format.
+/*! readU32FromCharChecked() :
+ * @return 0 if success, and store the result in *value.
* allows and interprets K, KB, KiB, M, MB and MiB suffix.
* Will also modify `*stringPtr`, advancing it to position where it stopped reading.
- * Note : function will exit() program if digit sequence overflows */
-static unsigned readU32FromChar(const char** stringPtr)
+ * @return 1 if an overflow error occurs */
+static int readU32FromCharChecked(const char** stringPtr, unsigned* value)
{
- const char errorMsg[] = "error: numeric value too large";
+ static unsigned const max = (((unsigned)(-1)) / 10) - 1;
unsigned result = 0;
while ((**stringPtr >='0') && (**stringPtr <='9')) {
- unsigned const max = (((unsigned)(-1)) / 10) - 1;
- if (result > max) errorOut(errorMsg);
+ if (result > max) return 1; // overflow error
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
}
if ((**stringPtr=='K') || (**stringPtr=='M')) {
unsigned const maxK = ((unsigned)(-1)) >> 10;
- if (result > maxK) errorOut(errorMsg);
+ if (result > maxK) return 1; // overflow error
result <<= 10;
if (**stringPtr=='M') {
- if (result > maxK) errorOut(errorMsg);
+ if (result > maxK) return 1; // overflow error
result <<= 10;
}
(*stringPtr)++; /* skip `K` or `M` */
if (**stringPtr=='i') (*stringPtr)++;
if (**stringPtr=='B') (*stringPtr)++;
}
+ *value = result;
+ return 0;
+}
+
+/*! readU32FromChar() :
+ * @return : unsigned integer value read from input in `char` format.
+ * allows and interprets K, KB, KiB, M, MB and MiB suffix.
+ * Will also modify `*stringPtr`, advancing it to position where it stopped reading.
+ * Note : function will exit() program if digit sequence overflows */
+static unsigned readU32FromChar(const char** stringPtr) {
+ static const char errorMsg[] = "error: numeric value too large";
+ unsigned result;
+ if (readU32FromCharChecked(stringPtr, &result)) { errorOut(errorMsg); }
return result;
}
@@ -391,7 +406,7 @@ static unsigned parseAdaptParameters(const char* stringPtr, int* adaptMinPtr, in
/** parseCompressionParameters() :
- * reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6") into *params
+ * reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6") into *params
* @return 1 means that compression parameters were correct
* @return 0 in case of malformed parameters
*/
@@ -402,20 +417,20 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi
if (longCommandWArg(&stringPtr, "chainLog=") || longCommandWArg(&stringPtr, "clog=")) { params->chainLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "hashLog=") || longCommandWArg(&stringPtr, "hlog=")) { params->hashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "searchLog=") || longCommandWArg(&stringPtr, "slog=")) { params->searchLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
- if (longCommandWArg(&stringPtr, "searchLength=") || longCommandWArg(&stringPtr, "slen=")) { params->searchLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+ if (longCommandWArg(&stringPtr, "minMatch=") || longCommandWArg(&stringPtr, "mml=")) { params->minMatch = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "targetLength=") || longCommandWArg(&stringPtr, "tlen=")) { params->targetLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)(readU32FromChar(&stringPtr)); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "overlapLog=") || longCommandWArg(&stringPtr, "ovlog=")) { g_overlapLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
- if (longCommandWArg(&stringPtr, "ldmHashLog=") || longCommandWArg(&stringPtr, "ldmhlog=")) { g_ldmHashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
- if (longCommandWArg(&stringPtr, "ldmSearchLength=") || longCommandWArg(&stringPtr, "ldmslen=")) { g_ldmMinMatch = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
- if (longCommandWArg(&stringPtr, "ldmBucketSizeLog=") || longCommandWArg(&stringPtr, "ldmblog=")) { g_ldmBucketSizeLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
- if (longCommandWArg(&stringPtr, "ldmHashEveryLog=") || longCommandWArg(&stringPtr, "ldmhevery=")) { g_ldmHashEveryLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+ if (longCommandWArg(&stringPtr, "ldmHashLog=") || longCommandWArg(&stringPtr, "lhlog=")) { g_ldmHashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+ if (longCommandWArg(&stringPtr, "ldmMinMatch=") || longCommandWArg(&stringPtr, "lmml=")) { g_ldmMinMatch = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+ if (longCommandWArg(&stringPtr, "ldmBucketSizeLog=") || longCommandWArg(&stringPtr, "lblog=")) { g_ldmBucketSizeLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+ if (longCommandWArg(&stringPtr, "ldmHashRateLog=") || longCommandWArg(&stringPtr, "lhrlog=")) { g_ldmHashRateLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
DISPLAYLEVEL(4, "invalid compression parameter \n");
return 0;
}
DISPLAYLEVEL(4, "windowLog=%d, chainLog=%d, hashLog=%d, searchLog=%d \n", params->windowLog, params->chainLog, params->hashLog, params->searchLog);
- DISPLAYLEVEL(4, "searchLength=%d, targetLength=%d, strategy=%d \n", params->searchLength, params->targetLength, params->strategy);
+ DISPLAYLEVEL(4, "minMatch=%d, targetLength=%d, strategy=%d \n", params->minMatch, params->targetLength, params->strategy);
if (stringPtr[0] != 0) return 0; /* check the end of string */
return 1;
}
@@ -450,6 +465,38 @@ static void printVersion(void)
#endif
}
+/* Environment variables for parameter setting */
+#define ENV_CLEVEL "ZSTD_CLEVEL"
+
+/* functions that pick up environment variables */
+static int init_cLevel(void) {
+ const char* const env = getenv(ENV_CLEVEL);
+ if (env) {
+ const char *ptr = env;
+ int sign = 1;
+ if (*ptr == '-') {
+ sign = -1;
+ ptr++;
+ } else if (*ptr == '+') {
+ ptr++;
+ }
+
+ if ((*ptr>='0') && (*ptr<='9')) {
+ unsigned absLevel;
+ if (readU32FromCharChecked(&ptr, &absLevel)) {
+ DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large\n", ENV_CLEVEL, env);
+ return ZSTDCLI_CLEVEL_DEFAULT;
+ } else if (*ptr == 0) {
+ return sign * absLevel;
+ }
+ }
+
+ DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: not a valid integer value\n", ENV_CLEVEL, env);
+ }
+
+ return ZSTDCLI_CLEVEL_DEFAULT;
+}
+
typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode;
#define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
@@ -475,6 +522,7 @@ int main(int argCount, const char* argv[])
adapt = 0,
adaptMin = MINCLEVEL,
adaptMax = MAXCLEVEL,
+ rsyncable = 0,
nextArgumentIsOutFileName = 0,
nextArgumentIsMaxDict = 0,
nextArgumentIsDictID = 0,
@@ -490,7 +538,7 @@ int main(int argCount, const char* argv[])
size_t blockSize = 0;
zstd_operation_mode operation = zom_compress;
ZSTD_compressionParameters compressionParams;
- int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
+ int cLevel;
int cLevelLast = -1000000000;
unsigned recursive = 0;
unsigned memLimit = 0;
@@ -525,6 +573,7 @@ int main(int argCount, const char* argv[])
if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); }
filenameTable[0] = stdinmark;
g_displayOut = stderr;
+ cLevel = init_cLevel();
programName = lastNameFromPath(programName);
#ifdef ZSTD_MULTITHREAD
nbWorkers = 1;
@@ -607,6 +656,8 @@ int main(int argCount, const char* argv[])
#ifdef ZSTD_LZ4COMPRESS
if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; FIO_setCompressionType(FIO_lz4Compression); continue; }
#endif
+ if (!strcmp(argument, "--rsyncable")) { rsyncable = 1; continue; }
+ if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; }
/* long commands with arguments */
#ifndef ZSTD_NODICT
@@ -937,8 +988,8 @@ int main(int argCount, const char* argv[])
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
benchParams.ldmBucketSizeLog = g_ldmBucketSizeLog;
}
- if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) {
- benchParams.ldmHashEveryLog = g_ldmHashEveryLog;
+ if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) {
+ benchParams.ldmHashRateLog = g_ldmHashRateLog;
}
if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
@@ -1048,10 +1099,11 @@ int main(int argCount, const char* argv[])
FIO_setLdmHashLog(g_ldmHashLog);
FIO_setLdmMinMatch(g_ldmMinMatch);
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(g_ldmBucketSizeLog);
- if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) FIO_setLdmHashEveryLog(g_ldmHashEveryLog);
+ if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(g_ldmHashRateLog);
FIO_setAdaptiveMode(adapt);
FIO_setAdaptMin(adaptMin);
FIO_setAdaptMax(adaptMax);
+ FIO_setRsyncable(rsyncable);
if (adaptMin > cLevel) cLevel = adaptMin;
if (adaptMax < cLevel) cLevel = adaptMax;
@@ -1060,7 +1112,7 @@ int main(int argCount, const char* argv[])
else
operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams);
#else
- (void)suffix; (void)adapt; (void)ultra; (void)cLevel; (void)ldmFlag; /* not used when ZSTD_NOCOMPRESS set */
+ (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; /* not used when ZSTD_NOCOMPRESS set */
DISPLAY("Compression not supported \n");
#endif
} else { /* decompression or test */
diff --git a/programs/zstdgrep b/programs/zstdgrep
index 9f871c03f1aa5..a10e0710a2967 100755
--- a/programs/zstdgrep
+++ b/programs/zstdgrep
@@ -31,94 +31,101 @@ grep_args=""
hyphen=0
silent=0
-prg=$(basename $0)
+prg=$(basename "$0")
# handle being called 'zegrep' or 'zfgrep'
-case ${prg} in
- *zegrep)
- grep_args="-E";;
- *zfgrep)
- grep_args="-F";;
+case "${prg}" in
+ *zegrep) grep_args="-E";;
+ *zfgrep) grep_args="-F";;
esac
# skip all options and pass them on to grep taking care of options
# with arguments, and if -e was supplied
-while [ $# -gt 0 -a ${endofopts} -eq 0 ]
-do
- case $1 in
+while [ "$#" -gt 0 ] && [ "${endofopts}" -eq 0 ]; do
+ case "$1" in
# from GNU grep-2.5.1 -- keep in sync!
- -[ABCDXdefm])
- if [ $# -lt 2 ]
- then
- echo "${prg}: missing argument for $1 flag" >&2
- exit 1
- fi
- case $1 in
- -e)
- pattern="$2"
- pattern_found=1
- shift 2
- break
- ;;
- *)
- ;;
- esac
- grep_args="${grep_args} $1 $2"
- shift 2
- ;;
- --)
- shift
- endofopts=1
- ;;
- -)
- hyphen=1
- shift
- ;;
- -h)
- silent=1
- shift
- ;;
- -*)
- grep_args="${grep_args} $1"
- shift
- ;;
- *)
- # pattern to grep for
- endofopts=1
- ;;
+ -[ABCDXdefm])
+ if [ "$#" -lt 2 ]; then
+ printf '%s: missing argument for %s flag\n' "${prg}" "$1" >&2
+ exit 1
+ fi
+ case "$1" in
+ -e)
+ pattern="$2"
+ pattern_found=1
+ shift 2
+ break
+ ;;
+ *)
+ ;;
+ esac
+ grep_args="${grep_args} $1 $2"
+ shift 2
+ ;;
+ --)
+ shift
+ endofopts=1
+ ;;
+ -)
+ hyphen=1
+ shift
+ ;;
+ -h)
+ silent=1
+ shift
+ ;;
+ -*)
+ grep_args="${grep_args} $1"
+ shift
+ ;;
+ *)
+ # pattern to grep for
+ endofopts=1
+ ;;
esac
done
# if no -e option was found, take next argument as grep-pattern
-if [ ${pattern_found} -lt 1 ]
-then
- if [ $# -ge 1 ]; then
- pattern="$1"
- shift
- elif [ ${hyphen} -gt 0 ]; then
- pattern="-"
+if [ "${pattern_found}" -lt 1 ]; then
+ if [ "$#" -ge 1 ]; then
+ pattern="$1"
+ shift
+ elif [ "${hyphen}" -gt 0 ]; then
+ pattern="-"
else
- echo "${prg}: missing pattern" >&2
- exit 1
+ printf '%s: missing pattern\n' "${prg}" >&2
+ exit 1
fi
fi
+EXIT_CODE=0
# call grep ...
-if [ $# -lt 1 ]
-then
+if [ "$#" -lt 1 ]; then
# ... on stdin
- ${zcat} -fq - | ${grep} ${grep_args} -- "${pattern}" -
+ set -f # Disable file name generation (globbing).
+ # shellcheck disable=SC2086
+ "${zcat}" -fq - | "${grep}" ${grep_args} -- "${pattern}" -
+ EXIT_CODE=$?
+ set +f
else
# ... on all files given on the command line
- if [ ${silent} -lt 1 -a $# -gt 1 ]; then
- grep_args="-H ${grep_args}"
+ if [ "${silent}" -lt 1 ] && [ "$#" -gt 1 ]; then
+ grep_args="-H ${grep_args}"
fi
- while [ $# -gt 0 ]
- do
- ${zcat} -fq -- "$1" | ${grep} --label="${1}" ${grep_args} -- "${pattern}" -
- shift
+ CUR_EXIT_CODE=0
+ EXIT_CODE=1
+ set -f
+ while [ "$#" -gt 0 ]; do
+ # shellcheck disable=SC2086
+ "${zcat}" -fq -- "$1" | "${grep}" --label="${1}" ${grep_args} -- "${pattern}" -
+ CUR_EXIT_CODE=$?
+ if [ "${CUR_EXIT_CODE}" -eq 0 ] && [ "${EXIT_CODE}" -ne 1 ]; then
+ EXIT_CODE=0
+ fi
+ shift
done
+ set +f
fi
-exit 0
+exit "${EXIT_CODE}"
diff --git a/programs/zstdgrep.1 b/programs/zstdgrep.1
index 716d28fc8e7ed..57bc14de3587d 100644
--- a/programs/zstdgrep.1
+++ b/programs/zstdgrep.1
@@ -1,5 +1,5 @@
.
-.TH "ZSTDGREP" "1" "October 2018" "zstd 1.3.7" "User Commands"
+.TH "ZSTDGREP" "1" "December 2018" "zstd 1.3.8" "User Commands"
.
.SH "NAME"
\fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files
diff --git a/programs/zstdless.1 b/programs/zstdless.1
index bf4965e7dfa6a..ff39742bf8a66 100644
--- a/programs/zstdless.1
+++ b/programs/zstdless.1
@@ -1,5 +1,5 @@
.
-.TH "ZSTDLESS" "1" "October 2018" "zstd 1.3.7" "User Commands"
+.TH "ZSTDLESS" "1" "December 2018" "zstd 1.3.8" "User Commands"
.
.SH "NAME"
\fBzstdless\fR \- view zstandard\-compressed files
generated by cgit v1.3 (git 2.53.0) at 2026-09-10 18:09:05 +0000