summaryrefslogtreecommitdiff
path: root/programs
diff options
context:
space:
mode:
authorConrad Meyer <cem@FreeBSD.org>2020-05-23 20:37:33 +0000
committerConrad Meyer <cem@FreeBSD.org>2020-05-23 20:37:33 +0000
commitbc64b5ce191d48b503e4fad8c0cefb774a2fa969 (patch)
tree9b41925d7159f1f57c1b59a1a5f887c80a57e999 /programs
parentea68403922c3b53b00fc999fcb3eaef1feb50177 (diff)
vendor/zstd/1.4.5vendor/zstd
Notes
Diffstat (limited to 'programs')
-rw-r--r--programs/Makefile93
-rw-r--r--programs/README.md78
-rw-r--r--programs/benchfn.c2
-rw-r--r--programs/benchfn.h2
-rw-r--r--programs/benchzstd.c13
-rw-r--r--programs/benchzstd.h4
-rw-r--r--programs/datagen.c4
-rw-r--r--programs/datagen.h2
-rw-r--r--programs/dibio.c6
-rw-r--r--programs/dibio.h4
-rw-r--r--programs/fileio.c397
-rw-r--r--programs/fileio.h6
-rw-r--r--programs/platform.h19
-rw-r--r--programs/timefn.c7
-rw-r--r--programs/timefn.h8
-rw-r--r--programs/util.c472
-rw-r--r--programs/util.h179
-rw-r--r--programs/zstd.1184
-rw-r--r--programs/zstd.1.md58
-rw-r--r--programs/zstdcli.c1055
-rwxr-xr-xprograms/zstdgrep6
-rw-r--r--programs/zstdgrep.12
-rw-r--r--programs/zstdless.12
23 files changed, 1630 insertions, 973 deletions
diff --git a/programs/Makefile b/programs/Makefile
index 64dcae0028e7..418ad4e6348e 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -1,10 +1,11 @@
# ################################################################
-# Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
+# Copyright (c) 2015-2020, Yann Collet, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
# ##########################################################################
# zstd : Command Line Utility, supporting gzip-like arguments
# zstd32 : Same as zstd, but forced to compile in 32-bits mode
@@ -42,9 +43,7 @@ else
ALIGN_LOOP =
endif
-CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
- -I$(ZSTDDIR)/dictBuilder \
- -DXXH_NAMESPACE=ZSTD_
+CPPFLAGS+= -DXXH_NAMESPACE=ZSTD_
ifeq ($(OS),Windows_NT) # MinGW assumed
CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting
endif
@@ -71,13 +70,15 @@ ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
ZSTDLEGACY_FILES += $(shell ls $(ZSTDDIR)/legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
endif
- CPPFLAGS += -I$(ZSTDDIR)/legacy
else
endif
# Sort files in alphabetical order for reproducible builds
ZSTDLIB_FILES := $(sort $(wildcard $(ZSTD_FILES)) $(wildcard $(ZSTDLEGACY_FILES)) $(wildcard $(ZDICT_FILES)))
+ZSTD_CLI_FILES := $(wildcard *.c)
+ZSTD_CLI_OBJ := $(patsubst %.c,%.o,$(ZSTD_CLI_FILES))
+
# Define *.exe as extension for Windows systems
ifneq (,$(filter Windows%,$(OS)))
EXT =.exe
@@ -94,9 +95,12 @@ endif
VOID = /dev/null
+# Make 4.3 doesn't support '\#' anymore (https://lwn.net/Articles/810071/)
+NUM_SYMBOL := \#
+
# thread detection
NO_THREAD_MSG := ==> no threads, building without multithreading support
-HAVE_PTHREAD := $(shell printf '\#include <pthread.h>\nint main(void) { return 0; }' > have_pthread.c && $(CC) $(FLAGS) -o have_pthread$(EXT) have_pthread.c -pthread 2> $(VOID) && rm have_pthread$(EXT) && echo 1 || echo 0; rm have_pthread.c)
+HAVE_PTHREAD := $(shell printf '$(NUM_SYMBOL)include <pthread.h>\nint main(void) { return 0; }' > have_pthread.c && $(CC) $(FLAGS) -o have_pthread$(EXT) have_pthread.c -pthread 2> $(VOID) && rm have_pthread$(EXT) && echo 1 || echo 0; rm have_pthread.c)
HAVE_THREAD := $(shell [ "$(HAVE_PTHREAD)" -eq "1" -o -n "$(filter Windows%,$(OS))" ] && echo 1 || echo 0)
ifeq ($(HAVE_THREAD), 1)
THREAD_MSG := ==> building with threading support
@@ -108,7 +112,7 @@ endif
# zlib detection
NO_ZLIB_MSG := ==> no zlib, building zstd without .gz support
-HAVE_ZLIB := $(shell printf '\#include <zlib.h>\nint main(void) { return 0; }' > have_zlib.c && $(CC) $(FLAGS) -o have_zlib$(EXT) have_zlib.c -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0; rm have_zlib.c)
+HAVE_ZLIB := $(shell printf '$(NUM_SYMBOL)include <zlib.h>\nint main(void) { return 0; }' > have_zlib.c && $(CC) $(FLAGS) -o have_zlib$(EXT) have_zlib.c -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0; rm have_zlib.c)
ifeq ($(HAVE_ZLIB), 1)
ZLIB_MSG := ==> building zstd with .gz compression support
ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS
@@ -119,7 +123,7 @@ endif
# lzma detection
NO_LZMA_MSG := ==> no liblzma, building zstd without .xz/.lzma support
-HAVE_LZMA := $(shell printf '\#include <lzma.h>\nint main(void) { return 0; }' > have_lzma.c && $(CC) $(FLAGS) -o have_lzma$(EXT) have_lzma.c -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0; rm have_lzma.c)
+HAVE_LZMA := $(shell printf '$(NUM_SYMBOL)include <lzma.h>\nint main(void) { return 0; }' > have_lzma.c && $(CC) $(FLAGS) -o have_lzma$(EXT) have_lzma.c -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0; rm have_lzma.c)
ifeq ($(HAVE_LZMA), 1)
LZMA_MSG := ==> building zstd with .xz/.lzma compression support
LZMACPP = -DZSTD_LZMACOMPRESS -DZSTD_LZMADECOMPRESS
@@ -130,7 +134,7 @@ endif
# lz4 detection
NO_LZ4_MSG := ==> no liblz4, building zstd without .lz4 support
-HAVE_LZ4 := $(shell printf '\#include <lz4frame.h>\n\#include <lz4.h>\nint main(void) { return 0; }' > have_lz4.c && $(CC) $(FLAGS) -o have_lz4$(EXT) have_lz4.c -llz4 2> $(VOID) && rm have_lz4$(EXT) && echo 1 || echo 0; rm have_lz4.c)
+HAVE_LZ4 := $(shell printf '$(NUM_SYMBOL)include <lz4frame.h>\n$(NUM_SYMBOL)include <lz4.h>\nint main(void) { return 0; }' > have_lz4.c && $(CC) $(FLAGS) -o have_lz4$(EXT) have_lz4.c -llz4 2> $(VOID) && rm have_lz4$(EXT) && echo 1 || echo 0; rm have_lz4.c)
ifeq ($(HAVE_LZ4), 1)
LZ4_MSG := ==> building zstd with .lz4 compression support
LZ4CPP = -DZSTD_LZ4COMPRESS -DZSTD_LZ4DECOMPRESS
@@ -158,22 +162,22 @@ default: zstd-release
all: zstd
.PHONY: allVariants
-allVariants: zstd zstd-compress zstd-decompress zstd-small zstd-nolegacy
+allVariants: zstd zstd-compress zstd-decompress zstd-small zstd-nolegacy zstd-dictBuilder
$(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)
zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP)
zstd : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD)
zstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
-zstd : $(ZSTDLIB_FILES) zstdcli.o util.o timefn.o fileio.o benchfn.o benchzstd.o datagen.o dibio.o
+ifneq (,$(filter Windows%,$(OS)))
+zstd : $(RES_FILE)
+endif
+zstd : $(ZSTDLIB_FILES) $(ZSTD_CLI_OBJ)
@echo "$(THREAD_MSG)"
@echo "$(ZLIB_MSG)"
@echo "$(LZMA_MSG)"
@echo "$(LZ4_MSG)"
-ifneq (,$(filter Windows%,$(OS)))
- windres/generate_res.bat
-endif
- $(CC) $(FLAGS) $^ $(RES_FILE) -o $@$(EXT) $(LDFLAGS)
+ $(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS)
.PHONY: zstd-release
zstd-release: DEBUGFLAGS := -DBACKTRACE_ENABLE=0
@@ -183,13 +187,15 @@ zstd-release: zstd
zstd32 : CPPFLAGS += $(THREAD_CPP)
zstd32 : LDFLAGS += $(THREAD_LD)
zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
-zstd32 : $(ZSTDLIB_FILES) zstdcli.c util.c timefn.c fileio.c benchfn.c benchzstd.c datagen.c dibio.c
ifneq (,$(filter Windows%,$(OS)))
- windres/generate_res.bat
+zstd32 : $(RES32_FILE)
endif
- $(CC) -m32 $(FLAGS) $^ $(RES32_FILE) -o $@$(EXT)
+zstd32 : $(ZSTDLIB_FILES) $(ZSTD_CLI_FILES)
+ $(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
-zstd-nolegacy : $(ZSTD_FILES) $(ZDICT_FILES) zstdcli.o util.o fileio.c benchfn.o benchzstd.o timefn.o datagen.o dibio.o
+## zstd-nolegacy: same scope as zstd, with just support of legacy formats removed
+zstd-nolegacy : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD)
+zstd-nolegacy : $(ZSTD_FILES) $(ZDICT_FILES) $(ZSTD_CLI_OBJ)
$(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS)
zstd-nomt : THREAD_CPP :=
@@ -207,7 +213,21 @@ zstd-noxz : LZMALD :=
zstd-noxz : LZMA_MSG := - xz/lzma support is disabled
zstd-noxz : zstd
+## zstd-dll: zstd executable linked to dynamic library libzstd (must already exist)
+# note : the following target doesn't link
+# because zstd uses non-public symbols from libzstd
+# such as XXH64 (for benchmark),
+# ZDICT_trainFromBuffer_unsafe_legacy (for dictionary builder)
+# and ZSTD_cycleLog (likely for --patch-from).
+# It's unclear at this stage if this is a scenario that must be supported
+.PHONY: zstd-dll
+zstd-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
+zstd-dll : ZSTDLIB_FILES =
+zstd-dll : $(ZSTD_CLI_OBJ)
+ $(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS)
+
+## zstd-pgo: zstd executable optimized with pgo. `gcc` only.
zstd-pgo :
$(MAKE) clean
$(MAKE) zstd MOREFLAGS=-fprofile-generate
@@ -218,9 +238,10 @@ zstd-pgo :
./zstd -b7i2 $(PROFILE_WITH)
./zstd -b5 $(PROFILE_WITH)
$(RM) zstd *.o $(ZSTDDECOMP_O) $(ZSTDDIR)/compress/*.o
+ case $(CC) in *clang*) if ! [ -e default.profdata ]; then llvm-profdata merge -output=default.profdata default*.profraw; fi ;; esac
$(MAKE) zstd MOREFLAGS=-fprofile-use
-# minimal target, with only zstd compression and decompression. no bench. no legacy.
+## zstd-small: minimal target, supporting only zstd compression and decompression. no bench. no legacy. no other format.
zstd-small: CFLAGS = -Os -s
zstd-frugal zstd-small: $(ZSTD_FILES) zstdcli.c util.c timefn.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT $^ -o $@$(EXT)
@@ -231,12 +252,25 @@ zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c util.c timefn
zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c util.c timefn.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT)
+## zstd-dictBuilder: executable supporting dictionary creation and compression (only)
+zstd-dictBuilder: CPPFLAGS += -DZSTD_NOBENCH -DZSTD_NODECOMPRESS
+zstd-dictBuilder: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) $(ZDICT_FILES) zstdcli.c util.c timefn.c fileio.c dibio.c
+ $(CC) $(FLAGS) $^ -o $@$(EXT)
+
zstdmt: zstd
ln -sf zstd zstdmt
.PHONY: generate_res
-generate_res:
- windres/generate_res.bat
+generate_res: $(RES64_FILE) $(RES32_FILE)
+
+ifneq (,$(filter Windows%,$(OS)))
+RC ?= windres
+# http://stackoverflow.com/questions/708238/how-do-i-add-an-icon-to-a-mingw-gcc-compiled-executable
+$(RES64_FILE): windres/zstd.rc
+ $(RC) -o $@ -I ../lib -I windres -i $< -O coff -F pe-x86-64
+$(RES32_FILE): windres/zstd.rc
+ $(RC) -o $@ -I ../lib -I windres -i $< -O coff -F pe-i386
+endif
.PHONY: clean
clean:
@@ -245,7 +279,7 @@ clean:
@$(RM) core *.o tmp* result* *.gcda dictionary *.zst \
zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \
zstd-small$(EXT) zstd-frugal$(EXT) zstd-nolegacy$(EXT) zstd4$(EXT) \
- *.gcda default.profraw have_zlib$(EXT)
+ zstd-dictBuilder$(EXT) *.gcda default*.profraw default.profdata have_zlib$(EXT)
@echo Cleaning completed
MD2ROFF = ronn
@@ -286,6 +320,7 @@ ifeq ($HAVE_COLORNEVER, 1)
EGREP_OPTIONS += --color=never
endif
EGREP = egrep $(EGREP_OPTIONS)
+AWK = awk
# Print a two column output of targets and their description. To add a target description, put a
# comment in the Makefile with the format "## <TARGET>: <DESCRIPTION>". For example:
@@ -294,14 +329,14 @@ EGREP = egrep $(EGREP_OPTIONS)
.PHONY: list
list:
@TARGETS=$$($(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null \
- | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' \
+ | $(AWK) -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' \
| $(EGREP) -v -e '^[^[:alnum:]]' | sort); \
{ \
printf "Target Name\tDescription\n"; \
printf "%0.s-" {1..16}; printf "\t"; printf "%0.s-" {1..40}; printf "\n"; \
for target in $$TARGETS; do \
line=$$($(EGREP) "^##[[:space:]]+$$target:" $(lastword $(MAKEFILE_LIST))); \
- description=$$(echo $$line | awk '{i=index($$0,":"); print substr($$0,i+1)}' | xargs); \
+ description=$$(echo $$line | $(AWK) '{i=index($$0,":"); print substr($$0,i+1)}' | xargs); \
printf "$$target\t$$description\n"; \
done \
} | column -t -s $$'\t'
@@ -342,10 +377,10 @@ INSTALL_MAN ?= $(INSTALL_DATA)
install: zstd
@echo Installing binaries
@$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MAN1DIR)/
- @$(INSTALL_PROGRAM) zstd $(DESTDIR)$(BINDIR)/zstd
- @ln -sf zstd $(DESTDIR)$(BINDIR)/zstdcat
- @ln -sf zstd $(DESTDIR)$(BINDIR)/unzstd
- @ln -sf zstd $(DESTDIR)$(BINDIR)/zstdmt
+ @$(INSTALL_PROGRAM) zstd$(EXT) $(DESTDIR)$(BINDIR)/zstd$(EXT)
+ @ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdcat$(EXT)
+ @ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/unzstd$(EXT)
+ @ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdmt$(EXT)
@$(INSTALL_SCRIPT) zstdless $(DESTDIR)$(BINDIR)/zstdless
@$(INSTALL_SCRIPT) zstdgrep $(DESTDIR)$(BINDIR)/zstdgrep
@echo Installing man pages
diff --git a/programs/README.md b/programs/README.md
index 7668d49a2073..53706de727b7 100644
--- a/programs/README.md
+++ b/programs/README.md
@@ -10,7 +10,7 @@ There are however other Makefile targets that create different variations of CLI
- `zstd-decompress` : version of CLI which can only decompress zstd format
-#### Compilation variables
+### Compilation variables
`zstd` scope can be altered by modifying the following `make` variables :
- __HAVE_THREAD__ : multithreading is automatically enabled when `pthread` is detected.
@@ -61,6 +61,24 @@ There are however other Makefile targets that create different variations of CLI
In which case, linking stage will fail if `lz4` library cannot be found.
This is useful to prevent silent feature disabling.
+- __ZSTD_NOBENCH__ : `zstd` cli will be compiled without its integrated benchmark module.
+ This can be useful to produce smaller binaries.
+ In this case, the corresponding unit can also be excluded from compilation target.
+
+- __ZSTD_NODICT__ : `zstd` cli will be compiled without support for the integrated dictionary builder.
+ This can be useful to produce smaller binaries.
+ In this case, the corresponding unit can also be excluded from compilation target.
+
+- __ZSTD_NOCOMPRESS__ : `zstd` cli will be compiled without support for compression.
+ The resulting binary will only be able to decompress files.
+ This can be useful to produce smaller binaries.
+ A corresponding `Makefile` target using this ability is `zstd-decompress`.
+
+- __ZSTD_NODECOMPRESS__ : `zstd` cli will be compiled without support for decompression.
+ The resulting binary will only be able to compress files.
+ This can be useful to produce smaller binaries.
+ A corresponding `Makefile` target using this ability is `zstd-compress`.
+
- __BACKTRACE__ : `zstd` can display a stack backtrace when execution
generates a runtime exception. By default, this feature may be
degraded/disabled on some platforms unless additional compiler directives are
@@ -69,11 +87,11 @@ There are however other Makefile targets that create different variations of CLI
Example : `make zstd BACKTRACE=1`
-#### Aggregation of parameters
+### Aggregation of parameters
CLI supports aggregation of parameters i.e. `-b1`, `-e18`, and `-i1` can be joined into `-b1e18i1`.
-#### Symlink shortcuts
+### Symlink shortcuts
It's possible to invoke `zstd` through a symlink.
When the name of the symlink has a specific value, it triggers an associated behavior.
- `zstdmt` : compress using all cores available on local system.
@@ -86,7 +104,7 @@ When the name of the symlink has a specific value, it triggers an associated beh
- `ungz`, `unxz` and `unlzma` will do the same, and will also remove source file by default (use `--keep` to preserve).
-#### Dictionary builder in Command Line Interface
+### Dictionary builder in Command Line Interface
Zstd offers a training mode, which can be used to tune the algorithm for a selected
type of data, by providing it with a few samples. The result of the training is stored
in a file selected with the `-o` option (default name is `dictionary`),
@@ -106,7 +124,7 @@ Usage of the dictionary builder and created dictionaries with CLI:
3. Decompress with the dictionary: `zstd --decompress FILE.zst -D dictionaryName`
-#### Benchmark in Command Line Interface
+### Benchmark in Command Line Interface
CLI includes in-memory compression benchmark module for zstd.
The benchmark is conducted using given filenames. The files are read into memory and joined together.
It makes benchmark more precise as it eliminates I/O overhead.
@@ -118,7 +136,7 @@ One can select compression levels starting from `-b` and ending with `-e`.
The `-i` parameter selects minimal time used for each of tested levels.
-#### Usage of Command Line Interface
+### Usage of Command Line Interface
The full list of options can be obtained with `-h` or `-H` parameter:
```
Usage :
@@ -142,23 +160,34 @@ Advanced arguments :
-q : suppress warnings; specify twice to suppress errors too
-c : force write to standard output, even if it is the console
-l : print information about zstd compressed files
+--exclude-compressed: only compress files that are not previously compressed
--ultra : enable levels beyond 19, up to 22 (requires more memory)
---long : enable long distance matching (requires more memory)
+--long[=#]: enable long distance matching with given window log (default: 27)
+--fast[=#]: switch to very fast compression levels (default: 1)
+--adapt : dynamically adapt compression level to I/O conditions
+--stream-size=# : optimize compression parameters for streaming input of given number of bytes
+--size-hint=# optimize compression parameters for streaming input of approximately this size
+--target-compressed-block-size=# : make compressed block near targeted size
+ -T# : spawns # compression threads (default: 1, 0==# cores)
+ -B# : select size of each job (default: 0==automatic)
+--rsyncable : compress using a rsync-friendly method (-B sets block size)
--no-dictID : don't write dictID into header (dictionary compression)
--[no-]check : integrity check (default: enabled)
+--[no-]compress-literals : force (un)compressed literals
-r : operate recursively on directories
+--output-dir-flat[=directory]: all resulting files stored into `directory`.
+--format=zstd : compress files to the .zst format (default)
--format=gzip : compress files to the .gz format
---format=xz : compress files to the .xz format
---format=lzma : compress files to the .lzma format
--test : test compressed file integrity
--[no-]sparse : sparse mode (default: disabled)
-M# : Set a memory usage limit for decompression
+--no-progress : do not display the progress bar
-- : All arguments after "--" are treated as files
Dictionary builder :
--train ## : create a dictionary from a training set of files
--train-cover[=k=#,d=#,steps=#,split=#,shrink[=#]] : use the cover algorithm with optional args
---train-fastcover[=k=#,d=#,f=#,steps=#,split=#,shrink[=#],accel=#] : use the fastcover algorithm with optional args
+--train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#,shrink[=#]] : use the fast cover algorithm with optional args
--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: 9)
-o file : `file` is dictionary name (default: dictionary)
--maxdict=# : limit dictionary to specified size (default: 112640)
@@ -172,16 +201,19 @@ Benchmark arguments :
--priority=rt : set process priority to real-time
```
-#### Restricted usage of Environment Variables
-Using environment variables to set parameters has security implications.
-Therefore, this avenue is intentionally restricted.
-Only `ZSTD_CLEVEL` is supported currently, for setting compression level.
-`ZSTD_CLEVEL` can be used to set the level between 1 and 19 (the "normal" range).
-If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a warning message.
-`ZSTD_CLEVEL` just replaces the default compression level (`3`).
-It can be overridden by corresponding command line arguments.
+### Passing parameters through Environment Variables
+`ZSTD_CLEVEL` can be used to modify the default compression level of `zstd`
+(usually set to `3`) to another value between 1 and 19 (the "normal" range).
+This can be useful when `zstd` CLI is invoked in a way that doesn't allow passing arguments.
+One such scenario is `tar --zstd`.
+As `ZSTD_CLEVEL` only replaces the default compression level,
+it can then be overridden by corresponding command line arguments.
+
+There is no "generic" way to pass "any kind of parameter" to `zstd` in a pass-through manner.
+Using environment variables for this purpose has security implications.
+Therefore, this avenue is intentionally restricted and only supports `ZSTD_CLEVEL`.
-#### Long distance matching mode
+### Long distance matching mode
The long distance matching mode, enabled with `--long`, is designed to improve
the compression ratio for files with long matches at a large distance (up to the
maximum window size, `128 MiB`) while still maintaining compression speed.
@@ -205,12 +237,12 @@ Compression Speed vs Ratio | Decompression Speed
| Method | Compression ratio | Compression speed | Decompression speed |
|:-------|------------------:|-------------------------:|---------------------------:|
-| `zstd -1` | `5.065` | `284.8 MB/s` | `759.3 MB/s` |
+| `zstd -1` | `5.065` | `284.8 MB/s` | `759.3 MB/s` |
| `zstd -5` | `5.826` | `124.9 MB/s` | `674.0 MB/s` |
| `zstd -10` | `6.504` | `29.5 MB/s` | `771.3 MB/s` |
| `zstd -1 --long` | `17.426` | `220.6 MB/s` | `1638.4 MB/s` |
-| `zstd -5 --long` | `19.661` | `165.5 MB/s` | `1530.6 MB/s`|
-| `zstd -10 --long`| `21.949` | `75.6 MB/s` | `1632.6 MB/s`|
+| `zstd -5 --long` | `19.661` | `165.5 MB/s` | `1530.6 MB/s` |
+| `zstd -10 --long`| `21.949` | `75.6 MB/s` | `1632.6 MB/s` |
On this file, the compression ratio improves significantly with minimal impact
on compression speed, and the decompression speed doubles.
@@ -233,7 +265,7 @@ The below table illustrates this on the [Silesia compression corpus].
| `zstd -10 --long`| `3.566` | `16.2 MB/s` | `415.7 MB/s` |
-#### zstdgrep
+### zstdgrep
`zstdgrep` is a utility which makes it possible to `grep` directly a `.zst` compressed file.
It's used the same way as normal `grep`, for example :
diff --git a/programs/benchfn.c b/programs/benchfn.c
index 2a51a34ff110..ed7273afb6e5 100644
--- a/programs/benchfn.c
+++ b/programs/benchfn.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/benchfn.h b/programs/benchfn.h
index 19e056581a5b..e555bbe6ae32 100644
--- a/programs/benchfn.h
+++ b/programs/benchfn.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/benchzstd.c b/programs/benchzstd.c
index 7439677c7f3e..77056203d55e 100644
--- a/programs/benchzstd.c
+++ b/programs/benchzstd.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -30,13 +30,13 @@
#include "timefn.h" /* UTIL_time_t */
#include "benchfn.h"
-#include "mem.h"
+#include "../lib/common/mem.h"
#define ZSTD_STATIC_LINKING_ONLY
-#include "zstd.h"
+#include "../lib/zstd.h"
#include "datagen.h" /* RDG_genBuffer */
-#include "xxhash.h"
+#include "../lib/common/xxhash.h"
#include "benchzstd.h"
-#include "zstd_errors.h"
+#include "../lib/common/zstd_errors.h"
/* *************************************
@@ -375,7 +375,6 @@ BMK_benchMemAdvancedNoAlloc(
resPtr += thisBlockSize;
remaining -= thisBlockSize;
if (adv->mode == BMK_decodeOnly) {
- assert(nbBlocks==0);
cSizes[nbBlocks] = thisBlockSize;
benchResult.cSize = thisBlockSize;
}
@@ -483,7 +482,7 @@ BMK_benchMemAdvancedNoAlloc(
{ int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
marks[markNb], displayName,
- (unsigned)srcSize, (unsigned)benchResult.cSize,
+ (unsigned)srcSize, (unsigned)cSize,
ratioAccuracy, ratio,
benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT,
(double)benchResult.dSpeed / MB_UNIT);
diff --git a/programs/benchzstd.h b/programs/benchzstd.h
index ef7d9fb11145..8c55b3c4f297 100644
--- a/programs/benchzstd.h
+++ b/programs/benchzstd.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -24,7 +24,7 @@ extern "C" {
/* === Dependencies === */
#include <stddef.h> /* size_t */
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
-#include "zstd.h" /* ZSTD_compressionParameters */
+#include "../lib/zstd.h" /* ZSTD_compressionParameters */
/* === Constants === */
diff --git a/programs/datagen.c b/programs/datagen.c
index ead9b2d2415a..4353b7ff9943 100644
--- a/programs/datagen.c
+++ b/programs/datagen.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -18,7 +18,7 @@
#include <stdlib.h> /* malloc, free */
#include <stdio.h> /* FILE, fwrite, fprintf */
#include <string.h> /* memcpy */
-#include "mem.h" /* U32 */
+#include "../lib/common/mem.h" /* U32 */
/*-************************************
diff --git a/programs/datagen.h b/programs/datagen.h
index 2fcc980e5e76..5a2682d8f9f6 100644
--- a/programs/datagen.h
+++ b/programs/datagen.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
diff --git a/programs/dibio.c b/programs/dibio.c
index ea4bb4bf1f30..463095a8e813 100644
--- a/programs/dibio.c
+++ b/programs/dibio.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -30,8 +30,8 @@
#include <assert.h>
#include "timefn.h" /* UTIL_time_t, UTIL_clockSpanMicro, UTIL_getTime */
-#include "mem.h" /* read */
-#include "error_private.h"
+#include "../lib/common/mem.h" /* read */
+#include "../lib/common/error_private.h"
#include "dibio.h"
diff --git a/programs/dibio.h b/programs/dibio.h
index ea163fe6afd9..682723d6a54b 100644
--- a/programs/dibio.h
+++ b/programs/dibio.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -19,7 +19,7 @@
* Dependencies
***************************************/
#define ZDICT_STATIC_LINKING_ONLY
-#include "zdict.h" /* ZDICT_params_t */
+#include "../lib/dictBuilder/zdict.h" /* ZDICT_params_t */
/*-*************************************
diff --git a/programs/fileio.c b/programs/fileio.c
index 9833767282ee..d72879d64eae 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -39,12 +39,13 @@
# include <io.h>
#endif
-#include "mem.h" /* U32, U64 */
+#include "../lib/common/mem.h" /* U32, U64 */
#include "fileio.h"
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
-#include "zstd.h"
-#include "zstd_errors.h" /* ZSTD_error_frameParameter_windowTooLarge */
+#include "../lib/zstd.h"
+#include "../lib/common/zstd_errors.h" /* ZSTD_error_frameParameter_windowTooLarge */
+#include "../lib/compress/zstd_compress_internal.h"
#if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS)
# include <zlib.h>
@@ -68,16 +69,11 @@
/*-*************************************
* Constants
***************************************/
-#define KB *(1<<10)
-#define MB *(1<<20)
-#define GB *(1U<<30)
-
#define ADAPT_WINDOWLOG_DEFAULT 23 /* 8 MB */
#define DICTSIZE_MAX (32 MB) /* protection against large input (attack scenario) */
#define FNSPACE 30
-
/*-*************************************
* Macros
***************************************/
@@ -99,7 +95,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
#define READY_FOR_UPDATE() (!g_display_prefs.noProgress && UTIL_clockSpanMicro(g_displayClock) > g_refreshRate)
#define DELAY_NEXT_UPDATE() { g_displayClock = UTIL_getTime(); }
#define DISPLAYUPDATE(l, ...) { \
- if (g_display_prefs.displayLevel>=l && !g_display_prefs.noProgress) { \
+ if (g_display_prefs.displayLevel>=l && !g_display_prefs.noProgress) { \
if (READY_FOR_UPDATE() || (g_display_prefs.displayLevel>=4)) { \
DELAY_NEXT_UPDATE(); \
DISPLAY(__VA_ARGS__); \
@@ -321,6 +317,8 @@ struct FIO_prefs_s {
int nbWorkers;
int excludeCompressedFiles;
+ int patchFromMode;
+ int contentSize;
};
@@ -487,6 +485,15 @@ void FIO_setLdmHashRateLog(FIO_prefs_t* const prefs, int ldmHashRateLog) {
prefs->ldmHashRateLog = ldmHashRateLog;
}
+void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value)
+{
+ prefs->patchFromMode = value != 0;
+}
+
+void FIO_setContentSize(FIO_prefs_t* const prefs, int value)
+{
+ prefs->contentSize = value != 0;
+}
/*-*************************************
* Functions
@@ -502,7 +509,7 @@ static int FIO_remove(const char* path)
#if defined(_WIN32) || defined(WIN32)
/* windows doesn't allow remove read-only files,
* so try to make it writable first */
- chmod(path, _S_IWRITE);
+ UTIL_chmod(path, _S_IWRITE);
#endif
return remove(path);
}
@@ -526,9 +533,7 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
}
if (!UTIL_isRegularFile(srcFileName)
-#ifndef _MSC_VER
- && !UTIL_isFIFO(srcFileName)
-#endif /* _MSC_VER */
+ && !UTIL_isFIFO(srcFileName)
) {
DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
srcFileName);
@@ -609,8 +614,11 @@ FIO_openDstFile(FIO_prefs_t* const prefs,
{ FILE* const f = fopen( dstFileName, "wb" );
if (f == NULL) {
DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno));
- } else if(srcFileName != NULL && strcmp (srcFileName, stdinmark)) {
- chmod(dstFileName, 00600);
+ } else if (srcFileName != NULL
+ && strcmp (srcFileName, stdinmark)
+ && strcmp(dstFileName, nulmark) ) {
+ /* reduce rights on newly created dst file while compression is ongoing */
+ UTIL_chmod(dstFileName, 00600);
}
return f;
}
@@ -623,7 +631,7 @@ FIO_openDstFile(FIO_prefs_t* const prefs,
* @return : loaded size
* if fileName==NULL, returns 0 and a NULL pointer
*/
-static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
+static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName, FIO_prefs_t* const prefs)
{
FILE* fileHandle;
U64 fileSize;
@@ -637,9 +645,12 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
if (fileHandle==NULL) EXM_THROW(31, "%s: %s", fileName, strerror(errno));
fileSize = UTIL_getFileSize(fileName);
- if (fileSize > DICTSIZE_MAX) {
- EXM_THROW(32, "Dictionary file %s is too large (> %u MB)",
- fileName, DICTSIZE_MAX >> 20); /* avoid extreme cases */
+ {
+ size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
+ if (fileSize > dictSizeMax) {
+ EXM_THROW(32, "Dictionary file %s is too large (> %u bytes)",
+ fileName, (unsigned)dictSizeMax); /* avoid extreme cases */
+ }
}
*bufferPtr = malloc((size_t)fileSize);
if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno));
@@ -742,6 +753,30 @@ FIO_createFilename_fromOutDir(const char* path, const char* outDirName, const si
return result;
}
+/* FIO_highbit64() :
+ * gives position of highest bit.
+ * note : only works for v > 0 !
+ */
+static unsigned FIO_highbit64(unsigned long long v)
+{
+ unsigned count = 0;
+ assert(v != 0);
+ v >>= 1;
+ while (v) { v >>= 1; count++; }
+ return count;
+}
+
+static void FIO_adjustMemLimitForPatchFromMode(FIO_prefs_t* const prefs,
+ unsigned long long const dictSize,
+ unsigned long long const maxSrcFileSize)
+{
+ unsigned long long maxSize = MAX(prefs->memLimit, MAX(dictSize, maxSrcFileSize));
+ assert(maxSize != UTIL_FILESIZE_UNKNOWN);
+ if (maxSize > UINT_MAX)
+ EXM_THROW(42, "Can't handle files larger than %u GB\n", UINT_MAX/(1 GB) + 1);
+ FIO_setMemLimit(prefs, (unsigned)maxSize);
+}
+
#ifndef ZSTD_NOCOMPRESS
/* **********************************************************************
@@ -754,13 +789,41 @@ typedef struct {
size_t srcBufferSize;
void* dstBuffer;
size_t dstBufferSize;
+ void* dictBuffer;
+ size_t dictBufferSize;
const char* dictFileName;
ZSTD_CStream* cctx;
} cRess_t;
+static void FIO_adjustParamsForPatchFromMode(FIO_prefs_t* const prefs,
+ ZSTD_compressionParameters* comprParams,
+ unsigned long long const dictSize,
+ unsigned long long const maxSrcFileSize,
+ int cLevel)
+{
+ unsigned const fileWindowLog = FIO_highbit64(maxSrcFileSize) + 1;
+ ZSTD_compressionParameters const cParams = ZSTD_getCParams(cLevel, (size_t)maxSrcFileSize, (size_t)dictSize);
+ FIO_adjustMemLimitForPatchFromMode(prefs, dictSize, maxSrcFileSize);
+ if (fileWindowLog > ZSTD_WINDOWLOG_MAX)
+ DISPLAYLEVEL(1, "Max window log exceeded by file (compression ratio will suffer)\n");
+ comprParams->windowLog = MIN(ZSTD_WINDOWLOG_MAX, fileWindowLog);
+ if (fileWindowLog > ZSTD_cycleLog(cParams.hashLog, cParams.strategy)) {
+ if (!prefs->ldmFlag)
+ DISPLAYLEVEL(1, "long mode automaticaly triggered\n");
+ FIO_setLdmFlag(prefs, 1);
+ }
+ if (cParams.strategy >= ZSTD_btopt) {
+ DISPLAYLEVEL(1, "[Optimal parser notes] Consider the following to improve patch size at the cost of speed:\n");
+ DISPLAYLEVEL(1, "- Use --single-thread mode in the zstd cli\n");
+ DISPLAYLEVEL(1, "- Set a larger targetLength (eg. --zstd=targetLength=4096)\n");
+ DISPLAYLEVEL(1, "- Set a larger chainLog (eg. --zstd=chainLog=%u)\n", ZSTD_CHAINLOG_MAX);
+ DISPLAYLEVEL(1, "Also consdier playing around with searchLog and hashLog\n");
+ }
+}
+
static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
- const char* dictFileName, int cLevel,
- ZSTD_compressionParameters comprParams) {
+ const char* dictFileName, unsigned long long const maxSrcFileSize,
+ int cLevel, ZSTD_compressionParameters comprParams) {
cRess_t ress;
memset(&ress, 0, sizeof(ress));
@@ -772,62 +835,68 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
ress.srcBufferSize = ZSTD_CStreamInSize();
ress.srcBuffer = malloc(ress.srcBufferSize);
ress.dstBufferSize = ZSTD_CStreamOutSize();
+
+ /* need to update memLimit before calling createDictBuffer
+ * because of memLimit check inside it */
+ if (prefs->patchFromMode)
+ FIO_adjustParamsForPatchFromMode(prefs, &comprParams, UTIL_getFileSize(dictFileName), maxSrcFileSize, cLevel);
ress.dstBuffer = malloc(ress.dstBufferSize);
+ ress.dictBufferSize = FIO_createDictBuffer(&ress.dictBuffer, dictFileName, prefs); /* works with dictFileName==NULL */
if (!ress.srcBuffer || !ress.dstBuffer)
EXM_THROW(31, "allocation error : not enough memory");
/* Advanced parameters, including dictionary */
- { void* dictBuffer;
- size_t const dictBuffSize = FIO_createDictBuffer(&dictBuffer, dictFileName); /* works with dictFileName==NULL */
- if (dictFileName && (dictBuffer==NULL))
- EXM_THROW(32, "allocation error : can't create dictBuffer");
- ress.dictFileName = dictFileName;
-
- if (prefs->adaptiveMode && !prefs->ldmFlag && !comprParams.windowLog)
- comprParams.windowLog = ADAPT_WINDOWLOG_DEFAULT;
-
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_contentSizeFlag, 1) ); /* always enable content size when available (note: supposed to be default) */
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_dictIDFlag, prefs->dictIDFlag) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_checksumFlag, prefs->checksumFlag) );
- /* compression level */
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, cLevel) );
- /* max compressed block size */
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetCBlockSize, (int)prefs->targetCBlockSize) );
- /* source size hint */
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_srcSizeHint, (int)prefs->srcSizeHint) );
- /* long distance matching */
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_enableLongDistanceMatching, prefs->ldmFlag) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashLog, prefs->ldmHashLog) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmMinMatch, prefs->ldmMinMatch) );
- if (prefs->ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) {
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmBucketSizeLog, prefs->ldmBucketSizeLog) );
- }
- if (prefs->ldmHashRateLog != FIO_LDM_PARAM_NOTSET) {
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashRateLog, prefs->ldmHashRateLog) );
- }
- /* compression parameters */
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_windowLog, (int)comprParams.windowLog) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_chainLog, (int)comprParams.chainLog) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_hashLog, (int)comprParams.hashLog) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_searchLog, (int)comprParams.searchLog) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_minMatch, (int)comprParams.minMatch) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetLength, (int)comprParams.targetLength) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_strategy, comprParams.strategy) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_literalCompressionMode, (int)prefs->literalCompressionMode) );
- /* multi-threading */
+ if (dictFileName && (ress.dictBuffer==NULL))
+ EXM_THROW(32, "allocation error : can't create dictBuffer");
+ ress.dictFileName = dictFileName;
+
+ if (prefs->adaptiveMode && !prefs->ldmFlag && !comprParams.windowLog)
+ comprParams.windowLog = ADAPT_WINDOWLOG_DEFAULT;
+
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_contentSizeFlag, prefs->contentSize) ); /* always enable content size when available (note: supposed to be default) */
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_dictIDFlag, prefs->dictIDFlag) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_checksumFlag, prefs->checksumFlag) );
+ /* compression level */
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, cLevel) );
+ /* max compressed block size */
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetCBlockSize, (int)prefs->targetCBlockSize) );
+ /* source size hint */
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_srcSizeHint, (int)prefs->srcSizeHint) );
+ /* long distance matching */
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_enableLongDistanceMatching, prefs->ldmFlag) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashLog, prefs->ldmHashLog) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmMinMatch, prefs->ldmMinMatch) );
+ if (prefs->ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) {
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmBucketSizeLog, prefs->ldmBucketSizeLog) );
+ }
+ if (prefs->ldmHashRateLog != FIO_LDM_PARAM_NOTSET) {
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashRateLog, prefs->ldmHashRateLog) );
+ }
+ /* compression parameters */
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_windowLog, (int)comprParams.windowLog) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_chainLog, (int)comprParams.chainLog) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_hashLog, (int)comprParams.hashLog) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_searchLog, (int)comprParams.searchLog) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_minMatch, (int)comprParams.minMatch) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetLength, (int)comprParams.targetLength) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_strategy, comprParams.strategy) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_literalCompressionMode, (int)prefs->literalCompressionMode) );
+ /* multi-threading */
#ifdef ZSTD_MULTITHREAD
- DISPLAYLEVEL(5,"set nb workers = %u \n", prefs->nbWorkers);
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_nbWorkers, prefs->nbWorkers) );
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, prefs->blockSize) );
- if (prefs->overlapLog != FIO_OVERLAP_LOG_NOTSET) {
- DISPLAYLEVEL(3,"set overlapLog = %u \n", prefs->overlapLog);
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_overlapLog, prefs->overlapLog) );
- }
- CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_rsyncable, prefs->rsyncable) );
+ DISPLAYLEVEL(5,"set nb workers = %u \n", prefs->nbWorkers);
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_nbWorkers, prefs->nbWorkers) );
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, prefs->blockSize) );
+ if (prefs->overlapLog != FIO_OVERLAP_LOG_NOTSET) {
+ DISPLAYLEVEL(3,"set overlapLog = %u \n", prefs->overlapLog);
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_overlapLog, prefs->overlapLog) );
+ }
+ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_rsyncable, prefs->rsyncable) );
#endif
- /* dictionary */
- CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) );
- free(dictBuffer);
+ /* dictionary */
+ if (prefs->patchFromMode) {
+ CHECK( ZSTD_CCtx_refPrefix(ress.cctx, ress.dictBuffer, ress.dictBufferSize) );
+ } else {
+ CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, ress.dictBuffer, ress.dictBufferSize) );
}
return ress;
@@ -837,6 +906,7 @@ static void FIO_freeCResources(cRess_t ress)
{
free(ress.srcBuffer);
free(ress.dstBuffer);
+ free(ress.dictBuffer);
ZSTD_freeCStream(ress.cctx); /* never fails */
}
@@ -1352,11 +1422,18 @@ FIO_compressFilename_internal(FIO_prefs_t* const prefs,
/* Status */
DISPLAYLEVEL(2, "\r%79s\r", "");
- DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n",
- srcFileName,
- (double)compressedfilesize / (readsize+(!readsize)/*avoid div by zero*/) * 100,
- (unsigned long long)readsize, (unsigned long long) compressedfilesize,
- dstFileName);
+ if (readsize == 0) {
+ DISPLAYLEVEL(2,"%-20s : (%6llu => %6llu bytes, %s) \n",
+ srcFileName,
+ (unsigned long long)readsize, (unsigned long long) compressedfilesize,
+ dstFileName);
+ } else {
+ DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n",
+ srcFileName,
+ (double)compressedfilesize / readsize * 100,
+ (unsigned long long)readsize, (unsigned long long) compressedfilesize,
+ dstFileName);
+ }
/* Elapsed Time and CPU Load */
{ clock_t const cpuEnd = clock();
@@ -1393,7 +1470,7 @@ static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs,
assert(ress.srcFile != NULL);
if (ress.dstFile == NULL) {
closeDstFile = 1;
- DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s", dstFileName);
+ DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s \n", dstFileName);
ress.dstFile = FIO_openDstFile(prefs, srcFileName, dstFileName);
if (ress.dstFile==NULL) return 1; /* could not open dstFileName */
/* Must only be added after FIO_openDstFile() succeeds.
@@ -1415,6 +1492,7 @@ static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs,
clearHandler();
+ DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: closing dst: %s \n", dstFileName);
if (fclose(dstFile)) { /* error closing dstFile */
DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno));
result=1;
@@ -1427,7 +1505,10 @@ static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs,
} else if ( strcmp(dstFileName, stdoutmark)
&& strcmp(dstFileName, nulmark)
&& transfer_permissions) {
+ DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: transfering permissions into dst: %s \n", dstFileName);
UTIL_setFileStat(dstFileName, &statbuf);
+ } else {
+ DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: do not transfer permissions into dst: %s \n", dstFileName);
}
}
@@ -1462,6 +1543,7 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs,
int compressionLevel)
{
int result;
+ DISPLAYLEVEL(6, "FIO_compressFilename_srcFile: %s \n", srcFileName);
/* ensure src is not a directory */
if (UTIL_isDirectory(srcFileName)) {
@@ -1507,9 +1589,9 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs,
int FIO_compressFilename(FIO_prefs_t* const prefs, const char* dstFileName,
const char* srcFileName, const char* dictFileName,
- int compressionLevel, ZSTD_compressionParameters comprParams)
+ int compressionLevel, ZSTD_compressionParameters comprParams)
{
- cRess_t const ress = FIO_createCResources(prefs, dictFileName, compressionLevel, comprParams);
+ cRess_t const ress = FIO_createCResources(prefs, dictFileName, UTIL_getFileSize(srcFileName), compressionLevel, comprParams);
int const result = FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
@@ -1557,6 +1639,16 @@ FIO_determineCompressedName(const char* srcFileName, const char* outDirName, con
return dstFileNameBuffer;
}
+static unsigned long long FIO_getLargestFileSize(const char** inFileNames, unsigned nbFiles)
+{
+ size_t i;
+ unsigned long long fileSize, maxFileSize = 0;
+ for (i = 0; i < nbFiles; i++) {
+ fileSize = UTIL_getFileSize(inFileNames[i]);
+ maxFileSize = fileSize > maxFileSize ? fileSize : maxFileSize;
+ }
+ return maxFileSize;
+}
/* FIO_compressMultipleFilenames() :
* compress nbFiles files
@@ -1572,7 +1664,9 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
ZSTD_compressionParameters comprParams)
{
int error = 0;
- cRess_t ress = FIO_createCResources(prefs, dictFileName, compressionLevel, comprParams);
+ cRess_t ress = FIO_createCResources(prefs, dictFileName,
+ FIO_getLargestFileSize(inFileNamesTable, nbFiles),
+ compressionLevel, comprParams);
/* init */
assert(outFileName != NULL || suffix != NULL);
@@ -1628,6 +1722,9 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi
dRess_t ress;
memset(&ress, 0, sizeof(ress));
+ if (prefs->patchFromMode)
+ FIO_adjustMemLimitForPatchFromMode(prefs, UTIL_getFileSize(dictFileName), 0 /* just use the dict size */);
+
/* Allocation */
ress.dctx = ZSTD_createDStream();
if (ress.dctx==NULL)
@@ -1642,7 +1739,7 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi
/* dictionary */
{ void* dictBuffer;
- size_t const dictBufferSize = FIO_createDictBuffer(&dictBuffer, dictFileName);
+ size_t const dictBufferSize = FIO_createDictBuffer(&dictBuffer, dictFileName, prefs);
CHECK( ZSTD_initDStream_usingDict(ress.dctx, dictBuffer, dictBufferSize) );
free(dictBuffer);
}
@@ -1659,18 +1756,19 @@ static void FIO_freeDResources(dRess_t ress)
/** FIO_fwriteSparse() :
-* @return : storedSkips, to be provided to next call to FIO_fwriteSparse() of LZ4IO_fwriteSparseEnd() */
+* @return : storedSkips,
+* argument for next call to FIO_fwriteSparse() or FIO_fwriteSparseEnd() */
static unsigned
-FIO_fwriteSparse(const FIO_prefs_t* const prefs,
- FILE* file,
+FIO_fwriteSparse(FILE* file,
const void* buffer, size_t bufferSize,
+ const FIO_prefs_t* const prefs,
unsigned storedSkips)
{
const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */
size_t bufferSizeT = bufferSize / sizeof(size_t);
const size_t* const bufferTEnd = bufferT + bufferSizeT;
const size_t* ptrT = bufferT;
- static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* 0-test re-attempted every 32 KB */
+ static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* check every 32 KB */
if (prefs->testMode) return 0; /* do not output anything in test mode */
@@ -1684,33 +1782,34 @@ FIO_fwriteSparse(const FIO_prefs_t* const prefs,
/* avoid int overflow */
if (storedSkips > 1 GB) {
- int const seekResult = LONG_SEEK(file, 1 GB, SEEK_CUR);
- if (seekResult != 0)
+ if (LONG_SEEK(file, 1 GB, SEEK_CUR) != 0)
EXM_THROW(91, "1 GB skip error (sparse file support)");
storedSkips -= 1 GB;
}
while (ptrT < bufferTEnd) {
- size_t seg0SizeT = segmentSizeT;
size_t nb0T;
- /* count leading zeros */
+ /* adjust last segment if < 32 KB */
+ size_t seg0SizeT = segmentSizeT;
if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT;
bufferSizeT -= seg0SizeT;
+
+ /* count leading zeroes */
for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ;
storedSkips += (unsigned)(nb0T * sizeof(size_t));
if (nb0T != seg0SizeT) { /* not all 0s */
- int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
- if (seekResult) EXM_THROW(92, "Sparse skip error ; try --no-sparse");
+ size_t const nbNon0ST = seg0SizeT - nb0T;
+ /* skip leading zeros */
+ if (LONG_SEEK(file, storedSkips, SEEK_CUR) != 0)
+ EXM_THROW(92, "Sparse skip error ; try --no-sparse");
storedSkips = 0;
- seg0SizeT -= nb0T;
- ptrT += nb0T;
- { size_t const sizeCheck = fwrite(ptrT, sizeof(size_t), seg0SizeT, file);
- if (sizeCheck != seg0SizeT)
- EXM_THROW(93, "Write error : cannot write decoded block : %s",
+ /* write the rest */
+ if (fwrite(ptrT + nb0T, sizeof(size_t), nbNon0ST, file) != nbNon0ST)
+ EXM_THROW(93, "Write error : cannot write decoded block : %s",
strerror(errno));
- } }
+ }
ptrT += seg0SizeT;
}
@@ -1719,20 +1818,20 @@ FIO_fwriteSparse(const FIO_prefs_t* const prefs,
/* size not multiple of sizeof(size_t) : implies end of block */
const char* const restStart = (const char*)bufferTEnd;
const char* restPtr = restStart;
- size_t restSize = bufferSize & maskT;
- const char* const restEnd = restStart + restSize;
+ const char* const restEnd = (const char*)buffer + bufferSize;
+ assert(restEnd > restStart && restEnd < restStart + sizeof(size_t));
for ( ; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ;
storedSkips += (unsigned) (restPtr - restStart);
if (restPtr != restEnd) {
- int seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
- if (seekResult)
- EXM_THROW(94, "Sparse skip error ; try --no-sparse");
+ /* not all remaining bytes are 0 */
+ size_t const restSize = (size_t)(restEnd - restPtr);
+ if (LONG_SEEK(file, storedSkips, SEEK_CUR) != 0)
+ EXM_THROW(92, "Sparse skip error ; try --no-sparse");
+ if (fwrite(restPtr, 1, restSize, file) != restSize)
+ EXM_THROW(95, "Write error : cannot write end of decoded block : %s",
+ strerror(errno));
storedSkips = 0;
- { size_t const sizeCheck = fwrite(restPtr, 1, (size_t)(restEnd - restPtr), file);
- if (sizeCheck != (size_t)(restEnd - restPtr))
- EXM_THROW(95, "Write error : cannot write decoded end of block : %s",
- strerror(errno));
- } } } }
+ } } }
return storedSkips;
}
@@ -1763,7 +1862,7 @@ static int FIO_passThrough(const FIO_prefs_t* const prefs,
size_t alreadyLoaded)
{
size_t const blockSize = MIN(64 KB, bufferSize);
- size_t readFromInput = 1;
+ size_t readFromInput;
unsigned storedSkips = 0;
/* assumption : ress->srcBufferLoaded bytes already loaded and stored within buffer */
@@ -1773,28 +1872,20 @@ static int FIO_passThrough(const FIO_prefs_t* const prefs,
return 1;
} }
- while (readFromInput) {
+ do {
readFromInput = fread(buffer, 1, blockSize, finput);
- storedSkips = FIO_fwriteSparse(prefs, foutput, buffer, readFromInput, storedSkips);
+ storedSkips = FIO_fwriteSparse(foutput, buffer, readFromInput, prefs, storedSkips);
+ } while (readFromInput == blockSize);
+ if (ferror(finput)) {
+ DISPLAYLEVEL(1, "Pass-through read error : %s\n", strerror(errno));
+ return 1;
}
+ assert(feof(finput));
FIO_fwriteSparseEnd(prefs, foutput, storedSkips);
return 0;
}
-/* FIO_highbit64() :
- * gives position of highest bit.
- * note : only works for v > 0 !
- */
-static unsigned FIO_highbit64(unsigned long long v)
-{
- unsigned count = 0;
- assert(v != 0);
- v >>= 1;
- while (v) { v >>= 1; count++; }
- return count;
-}
-
/* FIO_zstdErrorHelp() :
* detailed error message when requested window size is too large */
static void
@@ -1814,34 +1905,36 @@ FIO_zstdErrorHelp(const FIO_prefs_t* const prefs,
unsigned long long const windowSize = header.windowSize;
unsigned const windowLog = FIO_highbit64(windowSize) + ((windowSize & (windowSize - 1)) != 0);
assert(prefs->memLimit > 0);
- DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u\n",
+ DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u \n",
srcFileName, windowSize, prefs->memLimit);
if (windowLog <= ZSTD_WINDOWLOG_MAX) {
unsigned const windowMB = (unsigned)((windowSize >> 20) + ((windowSize & ((1 MB) - 1)) != 0));
assert(windowSize < (U64)(1ULL << 52)); /* ensure now overflow for windowMB */
- DISPLAYLEVEL(1, "%s : Use --long=%u or --memory=%uMB\n",
+ DISPLAYLEVEL(1, "%s : Use --long=%u or --memory=%uMB \n",
srcFileName, windowLog, windowMB);
return;
- }
- }
- DISPLAYLEVEL(1, "%s : Window log larger than ZSTD_WINDOWLOG_MAX=%u; not supported\n",
+ } }
+ DISPLAYLEVEL(1, "%s : Window log larger than ZSTD_WINDOWLOG_MAX=%u; not supported \n",
srcFileName, ZSTD_WINDOWLOG_MAX);
}
/** FIO_decompressFrame() :
* @return : size of decoded zstd frame, or an error code
-*/
+ */
#define FIO_ERROR_FRAME_DECODING ((unsigned long long)(-2))
static unsigned long long
-FIO_decompressZstdFrame(const FIO_prefs_t* const prefs,
- dRess_t* ress, FILE* finput,
- const char* srcFileName, U64 alreadyDecoded)
+FIO_decompressZstdFrame(dRess_t* ress, FILE* finput,
+ const FIO_prefs_t* const prefs,
+ const char* srcFileName,
+ U64 alreadyDecoded) /* for multi-frames streams */
{
U64 frameSize = 0;
U32 storedSkips = 0;
- size_t const srcFileLength = strlen(srcFileName);
- if (srcFileLength>20) srcFileName += srcFileLength-20; /* display last 20 characters only */
+ /* display last 20 characters only */
+ { size_t const srcFileLength = strlen(srcFileName);
+ if (srcFileLength>20) srcFileName += srcFileLength-20;
+ }
ZSTD_resetDStream(ress->dctx);
@@ -1866,7 +1959,7 @@ FIO_decompressZstdFrame(const FIO_prefs_t* const prefs,
}
/* Write block */
- storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, outBuff.pos, storedSkips);
+ storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, prefs, storedSkips);
frameSize += outBuff.pos;
DISPLAYUPDATE(2, "\r%-20.20s : %u MB... ",
srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) );
@@ -1900,8 +1993,8 @@ FIO_decompressZstdFrame(const FIO_prefs_t* const prefs,
#ifdef ZSTD_GZDECOMPRESS
static unsigned long long
-FIO_decompressGzFrame(const FIO_prefs_t* const prefs,
- dRess_t* ress, FILE* srcFile,
+FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile,
+ const FIO_prefs_t* const prefs,
const char* srcFileName)
{
unsigned long long outFileSize = 0;
@@ -1943,7 +2036,7 @@ FIO_decompressGzFrame(const FIO_prefs_t* const prefs,
}
{ size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
if (decompBytes) {
- storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, decompBytes, storedSkips);
+ storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, decompBytes, prefs, storedSkips);
outFileSize += decompBytes;
strm.next_out = (Bytef*)ress->dstBuffer;
strm.avail_out = (uInt)ress->dstBufferSize;
@@ -1968,8 +2061,8 @@ FIO_decompressGzFrame(const FIO_prefs_t* const prefs,
#ifdef ZSTD_LZMADECOMPRESS
static unsigned long long
-FIO_decompressLzmaFrame(const FIO_prefs_t* const prefs,
- dRess_t* ress, FILE* srcFile,
+FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile,
+ const FIO_prefs_t* const prefs,
const char* srcFileName, int plain_lzma)
{
unsigned long long outFileSize = 0;
@@ -2020,7 +2113,7 @@ FIO_decompressLzmaFrame(const FIO_prefs_t* const prefs,
}
{ size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
if (decompBytes) {
- storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, decompBytes, storedSkips);
+ storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, decompBytes, prefs, storedSkips);
outFileSize += decompBytes;
strm.next_out = (BYTE*)ress->dstBuffer;
strm.avail_out = ress->dstBufferSize;
@@ -2039,8 +2132,8 @@ FIO_decompressLzmaFrame(const FIO_prefs_t* const prefs,
#ifdef ZSTD_LZ4DECOMPRESS
static unsigned long long
-FIO_decompressLz4Frame(const FIO_prefs_t* const prefs,
- dRess_t* ress, FILE* srcFile,
+FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile,
+ const FIO_prefs_t* const prefs,
const char* srcFileName)
{
unsigned long long filesize = 0;
@@ -2092,7 +2185,7 @@ FIO_decompressLz4Frame(const FIO_prefs_t* const prefs,
/* Write Block */
if (decodedBytes) {
- storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, decodedBytes, storedSkips);
+ storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, decodedBytes, prefs, storedSkips);
filesize += decodedBytes;
DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20));
}
@@ -2127,8 +2220,8 @@ FIO_decompressLz4Frame(const FIO_prefs_t* const prefs,
* @return : 0 : OK
* 1 : error
*/
-static int FIO_decompressFrames(const FIO_prefs_t* const prefs,
- dRess_t ress, FILE* srcFile,
+static int FIO_decompressFrames(dRess_t ress, FILE* srcFile,
+ const FIO_prefs_t* const prefs,
const char* dstFileName, const char* srcFileName)
{
unsigned readSomething = 0;
@@ -2156,12 +2249,12 @@ static int FIO_decompressFrames(const FIO_prefs_t* const prefs,
return 1;
}
if (ZSTD_isFrame(buf, ress.srcBufferLoaded)) {
- unsigned long long const frameSize = FIO_decompressZstdFrame(prefs, &ress, srcFile, srcFileName, filesize);
+ unsigned long long const frameSize = FIO_decompressZstdFrame(&ress, srcFile, prefs, srcFileName, filesize);
if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
filesize += frameSize;
} else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */
#ifdef ZSTD_GZDECOMPRESS
- unsigned long long const frameSize = FIO_decompressGzFrame(prefs, &ress, srcFile, srcFileName);
+ unsigned long long const frameSize = FIO_decompressGzFrame(&ress, srcFile, prefs, srcFileName);
if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
filesize += frameSize;
#else
@@ -2171,7 +2264,7 @@ static int FIO_decompressFrames(const FIO_prefs_t* const prefs,
} else if ((buf[0] == 0xFD && buf[1] == 0x37) /* xz magic number */
|| (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */
#ifdef ZSTD_LZMADECOMPRESS
- unsigned long long const frameSize = FIO_decompressLzmaFrame(prefs, &ress, srcFile, srcFileName, buf[0] != 0xFD);
+ unsigned long long const frameSize = FIO_decompressLzmaFrame(&ress, srcFile, prefs, srcFileName, buf[0] != 0xFD);
if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
filesize += frameSize;
#else
@@ -2180,7 +2273,7 @@ static int FIO_decompressFrames(const FIO_prefs_t* const prefs,
#endif
} else if (MEM_readLE32(buf) == LZ4_MAGICNUMBER) {
#ifdef ZSTD_LZ4DECOMPRESS
- unsigned long long const frameSize = FIO_decompressLz4Frame(prefs, &ress, srcFile, srcFileName);
+ unsigned long long const frameSize = FIO_decompressLz4Frame(&ress, srcFile, prefs, srcFileName);
if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
filesize += frameSize;
#else
@@ -2237,7 +2330,7 @@ static int FIO_decompressDstFile(FIO_prefs_t* const prefs,
transfer_permissions = 1;
}
- result = FIO_decompressFrames(prefs, ress, srcFile, dstFileName, srcFileName);
+ result = FIO_decompressFrames(ress, srcFile, prefs, dstFileName, srcFileName);
if (releaseDstFile) {
FILE* const dstFile = ress.dstFile;
@@ -2718,7 +2811,7 @@ FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel)
displayInfo(inFileName, &info, displayLevel);
*total = FIO_addFInfo(*total, info);
assert(error == info_success || error == info_frame_error);
- return error;
+ return (int)error;
}
}
diff --git a/programs/fileio.h b/programs/fileio.h
index a7da089f67da..2fbf01f82824 100644
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -13,7 +13,7 @@
#define FILEIO_H_23981798732
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
-#include "zstd.h" /* ZSTD_* */
+#include "../lib/zstd.h" /* ZSTD_* */
#if defined (__cplusplus)
extern "C" {
@@ -94,6 +94,8 @@ void FIO_setLiteralCompressionMode(
void FIO_setNoProgress(unsigned noProgress);
void FIO_setNotificationLevel(int level);
void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles);
+void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value);
+void FIO_setContentSize(FIO_prefs_t* const prefs, int value);
/*-*************************************
* Single File functions
diff --git a/programs/platform.h b/programs/platform.h
index 5934e59cf12d..2b4b9f2d8677 100644
--- a/programs/platform.h
+++ b/programs/platform.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2020, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -90,7 +90,7 @@ extern "C" {
&& ( defined(__unix__) || defined(__unix) \
|| defined(__midipix__) || defined(__VMS) || defined(__HAIKU__) )
-# if defined(__linux__) || defined(__linux)
+# if defined(__linux__) || defined(__linux) || defined(__CYGWIN__)
# ifndef _POSIX_C_SOURCE
# define _POSIX_C_SOURCE 200809L /* feature test macro : https://www.gnu.org/software/libc/manual/html_node/Feature-Test-Macros.html */
# endif
@@ -109,16 +109,25 @@ extern "C" {
#endif /* PLATFORM_POSIX_VERSION */
+#if PLATFORM_POSIX_VERSION > 1
+ /* glibc < 2.26 may not expose struct timespec def without this.
+ * See issue #1920. */
+# ifndef _ATFILE_SOURCE
+# define _ATFILE_SOURCE
+# endif
+#endif
+
+
/*-*********************************************
* Detect if isatty() and fileno() are available
************************************************/
#if (defined(__linux__) && (PLATFORM_POSIX_VERSION > 1)) \
|| (PLATFORM_POSIX_VERSION >= 200112L) \
- || defined(__DJGPP__) \
- || defined(__MSYS__)
+ || defined(__DJGPP__)
# include <unistd.h> /* isatty */
+# include <stdio.h> /* fileno */
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
-#elif defined(MSDOS) || defined(OS2) || defined(__CYGWIN__)
+#elif defined(MSDOS) || defined(OS2)
# include <io.h> /* _isatty */
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
#elif defined(WIN32) || defined(_WIN32)
diff --git a/programs/timefn.c b/programs/timefn.c
index 096e1910bf4b..95460d0d971d 100644
--- a/programs/timefn.c
+++ b/programs/timefn.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2019-2020, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -82,9 +82,10 @@ PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
}
-
+/* C11 requires timespec_get, but FreeBSD 11 lacks it, while still claiming C11 compliance.
+ Android also lacks it but does define TIME_UTC. */
#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
- && defined(TIME_UTC) /* C11 requires timespec_get, but FreeBSD 11 lacks it, while still claiming C11 compliance */
+ && defined(TIME_UTC) && !defined(__ANDROID__)
#include <stdlib.h> /* abort */
#include <stdio.h> /* perror */
diff --git a/programs/timefn.h b/programs/timefn.h
index 2db3765b9308..eb3c130934eb 100644
--- a/programs/timefn.h
+++ b/programs/timefn.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -41,7 +41,7 @@ extern "C" {
******************************************/
#if defined(_WIN32) /* Windows */
- #include <Windows.h> /* LARGE_INTEGER */
+ #include <windows.h> /* LARGE_INTEGER */
typedef LARGE_INTEGER UTIL_time_t;
#define UTIL_TIME_INITIALIZER { { 0, 0 } }
@@ -51,8 +51,10 @@ extern "C" {
typedef PTime UTIL_time_t;
#define UTIL_TIME_INITIALIZER 0
+/* C11 requires timespec_get, but FreeBSD 11 lacks it, while still claiming C11 compliance.
+ Android also lacks it but does define TIME_UTC. */
#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
- && defined(TIME_UTC) /* C11 requires timespec_get, but FreeBSD 11 lacks it, while still claiming C11 compliance */
+ && defined(TIME_UTC) && !defined(__ANDROID__)
typedef struct timespec UTIL_time_t;
#define UTIL_TIME_INITIALIZER { 0, 0 }
diff --git a/programs/util.c b/programs/util.c
index 5d15450d2e13..ab1abd3b1862 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2020, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -17,13 +17,89 @@ extern "C" {
* Dependencies
******************************************/
#include "util.h" /* note : ensure that platform.h is included first ! */
+#include <stdlib.h> /* malloc, realloc, free */
+#include <stdio.h> /* fprintf */
+#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC, nanosleep */
#include <errno.h>
#include <assert.h>
+#if defined(_WIN32)
+# include <sys/utime.h> /* utime */
+# include <io.h> /* _chmod */
+#else
+# include <unistd.h> /* chown, stat */
+# if PLATFORM_POSIX_VERSION < 200809L
+# include <utime.h> /* utime */
+# else
+# include <fcntl.h> /* AT_FDCWD */
+# include <sys/stat.h> /* utimensat */
+# endif
+#endif
+
#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__)
#include <direct.h> /* needed for _mkdir in windows */
#endif
+#if defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */
+# include <dirent.h> /* opendir, readdir */
+# include <string.h> /* strerror, memcpy */
+#endif /* #ifdef _WIN32 */
+
+
+/*-****************************************
+* Internal Macros
+******************************************/
+
+/* CONTROL is almost like an assert(), but is never disabled.
+ * It's designed for failures that may happen rarely,
+ * but we don't want to maintain a specific error code path for them,
+ * such as a malloc() returning NULL for example.
+ * Since it's always active, this macro can trigger side effects.
+ */
+#define CONTROL(c) { \
+ if (!(c)) { \
+ UTIL_DISPLAYLEVEL(1, "Error : %s, %i : %s", \
+ __FILE__, __LINE__, #c); \
+ exit(1); \
+} }
+
+/* console log */
+#define UTIL_DISPLAY(...) fprintf(stderr, __VA_ARGS__)
+#define UTIL_DISPLAYLEVEL(l, ...) { if (g_utilDisplayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } }
+
+/* A modified version of realloc().
+ * If UTIL_realloc() fails the original block is freed.
+ */
+UTIL_STATIC void* UTIL_realloc(void *ptr, size_t size)
+{
+ void *newptr = realloc(ptr, size);
+ if (newptr) return newptr;
+ free(ptr);
+ return NULL;
+}
+
+#if defined(_MSC_VER)
+ #define chmod _chmod
+#endif
+
+
+/*-****************************************
+* Console log
+******************************************/
+int g_utilDisplayLevel;
+
+
+/*-*************************************
+* Constants
+***************************************/
+#define LIST_SIZE_INCREASE (8*1024)
+#define MAX_FILE_OF_FILE_NAMES_SIZE (1<<20)*50
+
+
+/*-*************************************
+* Functions
+***************************************/
+
int UTIL_fileExist(const char* filename)
{
stat_t statbuf;
@@ -54,6 +130,13 @@ int UTIL_getFileStat(const char* infilename, stat_t *statbuf)
return 1;
}
+/* like chmod, but avoid changing permission of /dev/null */
+int UTIL_chmod(char const* filename, mode_t permissions)
+{
+ if (!strcmp(filename, "/dev/null")) return 0; /* pretend success, but don't change anything */
+ return chmod(filename, permissions);
+}
+
int UTIL_setFileStat(const char *filename, stat_t *statbuf)
{
int res = 0;
@@ -62,41 +145,44 @@ int UTIL_setFileStat(const char *filename, stat_t *statbuf)
return -1;
/* set access and modification times */
-#if defined(_WIN32) || (PLATFORM_POSIX_VERSION < 200809L)
- {
- struct utimbuf timebuf;
- timebuf.actime = time(NULL);
- timebuf.modtime = statbuf->st_mtime;
- res += utime(filename, &timebuf);
- }
-#else
+ /* We check that st_mtime is a macro here in order to give us confidence
+ * that struct stat has a struct timespec st_mtim member. We need this
+ * check because there are some platforms that claim to be POSIX 2008
+ * compliant but which do not have st_mtim... */
+#if (PLATFORM_POSIX_VERSION >= 200809L) && defined(st_mtime)
{
/* (atime, mtime) */
struct timespec timebuf[2] = { {0, UTIME_NOW} };
timebuf[1] = statbuf->st_mtim;
res += utimensat(AT_FDCWD, filename, timebuf, 0);
}
+#else
+ {
+ struct utimbuf timebuf;
+ timebuf.actime = time(NULL);
+ timebuf.modtime = statbuf->st_mtime;
+ res += utime(filename, &timebuf);
+ }
#endif
#if !defined(_WIN32)
res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */
#endif
- res += chmod(filename, statbuf->st_mode & 07777); /* Copy file permissions */
+ res += UTIL_chmod(filename, statbuf->st_mode & 07777); /* Copy file permissions */
errno = 0;
return -res; /* number of errors is returned */
}
-U32 UTIL_isDirectory(const char* infilename)
+int UTIL_isDirectory(const char* infilename)
{
- int r;
stat_t statbuf;
#if defined(_MSC_VER)
- r = _stat64(infilename, &statbuf);
+ int const r = _stat64(infilename, &statbuf);
if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
#else
- r = stat(infilename, &statbuf);
+ int const r = stat(infilename, &statbuf);
if (!r && S_ISDIR(statbuf.st_mode)) return 1;
#endif
return 0;
@@ -126,28 +212,25 @@ int UTIL_isSameFile(const char* fName1, const char* fName2)
#endif
}
-#ifndef _MSC_VER
-/* Using this to distinguish named pipes */
-U32 UTIL_isFIFO(const char* infilename)
+/* UTIL_isFIFO : distinguish named pipes */
+int UTIL_isFIFO(const char* infilename)
{
/* macro guards, as defined in : https://linux.die.net/man/2/lstat */
#if PLATFORM_POSIX_VERSION >= 200112L
stat_t statbuf;
- int r = UTIL_getFileStat(infilename, &statbuf);
+ int const r = UTIL_getFileStat(infilename, &statbuf);
if (!r && S_ISFIFO(statbuf.st_mode)) return 1;
#endif
(void)infilename;
return 0;
}
-#endif
-U32 UTIL_isLink(const char* infilename)
+int UTIL_isLink(const char* infilename)
{
/* macro guards, as defined in : https://linux.die.net/man/2/lstat */
#if PLATFORM_POSIX_VERSION >= 200112L
- int r;
stat_t statbuf;
- r = lstat(infilename, &statbuf);
+ int const r = lstat(infilename, &statbuf);
if (!r && S_ISLNK(statbuf.st_mode)) return 1;
#endif
(void)infilename;
@@ -176,28 +259,226 @@ U64 UTIL_getFileSize(const char* infilename)
}
-U64 UTIL_getTotalFileSize(const char* const * const fileNamesTable, unsigned nbFiles)
+U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles)
{
U64 total = 0;
- int error = 0;
unsigned n;
for (n=0; n<nbFiles; n++) {
U64 const size = UTIL_getFileSize(fileNamesTable[n]);
- error |= (size == UTIL_FILESIZE_UNKNOWN);
+ if (size == UTIL_FILESIZE_UNKNOWN) return UTIL_FILESIZE_UNKNOWN;
total += size;
}
- return error ? UTIL_FILESIZE_UNKNOWN : total;
+ return total;
+}
+
+
+/* condition : @file must be valid, and not have reached its end.
+ * @return : length of line written into @buf, ended with `\0` instead of '\n',
+ * or 0, if there is no new line */
+static size_t readLineFromFile(char* buf, size_t len, FILE* file)
+{
+ assert(!feof(file));
+ /* Work around Cygwin problem when len == 1 it returns NULL. */
+ if (len <= 1) return 0;
+ CONTROL( fgets(buf, (int) len, file) );
+ { size_t linelen = strlen(buf);
+ if (strlen(buf)==0) return 0;
+ if (buf[linelen-1] == '\n') linelen--;
+ buf[linelen] = '\0';
+ return linelen+1;
+ }
+}
+
+/* Conditions :
+ * size of @inputFileName file must be < @dstCapacity
+ * @dst must be initialized
+ * @return : nb of lines
+ * or -1 if there's an error
+ */
+static int
+readLinesFromFile(void* dst, size_t dstCapacity,
+ const char* inputFileName)
+{
+ int nbFiles = 0;
+ size_t pos = 0;
+ char* const buf = (char*)dst;
+ FILE* const inputFile = fopen(inputFileName, "r");
+
+ assert(dst != NULL);
+
+ if(!inputFile) {
+ if (g_utilDisplayLevel >= 1) perror("zstd:util:readLinesFromFile");
+ return -1;
+ }
+
+ while ( !feof(inputFile) ) {
+ size_t const lineLength = readLineFromFile(buf+pos, dstCapacity-pos, inputFile);
+ if (lineLength == 0) break;
+ assert(pos + lineLength < dstCapacity);
+ pos += lineLength;
+ ++nbFiles;
+ }
+
+ CONTROL( fclose(inputFile) == 0 );
+
+ return nbFiles;
+}
+
+/*Note: buf is not freed in case function successfully created table because filesTable->fileNames[0] = buf*/
+FileNamesTable*
+UTIL_createFileNamesTable_fromFileName(const char* inputFileName)
+{
+ size_t nbFiles = 0;
+ char* buf;
+ size_t bufSize;
+ size_t pos = 0;
+
+ if (!UTIL_fileExist(inputFileName) || !UTIL_isRegularFile(inputFileName))
+ return NULL;
+
+ { U64 const inputFileSize = UTIL_getFileSize(inputFileName);
+ if(inputFileSize > MAX_FILE_OF_FILE_NAMES_SIZE)
+ return NULL;
+ bufSize = (size_t)(inputFileSize + 1); /* (+1) to add '\0' at the end of last filename */
+ }
+
+ buf = (char*) malloc(bufSize);
+ CONTROL( buf != NULL );
+
+ { int const ret_nbFiles = readLinesFromFile(buf, bufSize, inputFileName);
+
+ if (ret_nbFiles <= 0) {
+ free(buf);
+ return NULL;
+ }
+ nbFiles = (size_t)ret_nbFiles;
+ }
+
+ { const char** filenamesTable = (const char**) malloc(nbFiles * sizeof(*filenamesTable));
+ CONTROL(filenamesTable != NULL);
+
+ { size_t fnb;
+ for (fnb = 0, pos = 0; fnb < nbFiles; fnb++) {
+ filenamesTable[fnb] = buf+pos;
+ pos += strlen(buf+pos)+1; /* +1 for the finishing `\0` */
+ } }
+ assert(pos <= bufSize);
+
+ return UTIL_assembleFileNamesTable(filenamesTable, nbFiles, buf);
+ }
+}
+
+static FileNamesTable*
+UTIL_assembleFileNamesTable2(const char** filenames, size_t tableSize, size_t tableCapacity, char* buf)
+{
+ FileNamesTable* const table = (FileNamesTable*) malloc(sizeof(*table));
+ CONTROL(table != NULL);
+ table->fileNames = filenames;
+ table->buf = buf;
+ table->tableSize = tableSize;
+ table->tableCapacity = tableCapacity;
+ return table;
+}
+
+FileNamesTable*
+UTIL_assembleFileNamesTable(const char** filenames, size_t tableSize, char* buf)
+{
+ return UTIL_assembleFileNamesTable2(filenames, tableSize, tableSize, buf);
+}
+
+void UTIL_freeFileNamesTable(FileNamesTable* table)
+{
+ if (table==NULL) return;
+ free((void*)table->fileNames);
+ free(table->buf);
+ free(table);
+}
+
+FileNamesTable* UTIL_allocateFileNamesTable(size_t tableSize)
+{
+ const char** const fnTable = (const char**)malloc(tableSize * sizeof(*fnTable));
+ FileNamesTable* fnt;
+ if (fnTable==NULL) return NULL;
+ fnt = UTIL_assembleFileNamesTable(fnTable, tableSize, NULL);
+ fnt->tableSize = 0; /* the table is empty */
+ return fnt;
+}
+
+void UTIL_refFilename(FileNamesTable* fnt, const char* filename)
+{
+ assert(fnt->tableSize < fnt->tableCapacity);
+ fnt->fileNames[fnt->tableSize] = filename;
+ fnt->tableSize++;
+}
+
+static size_t getTotalTableSize(FileNamesTable* table)
+{
+ size_t fnb = 0, totalSize = 0;
+ for(fnb = 0 ; fnb < table->tableSize && table->fileNames[fnb] ; ++fnb) {
+ totalSize += strlen(table->fileNames[fnb]) + 1; /* +1 to add '\0' at the end of each fileName */
+ }
+ return totalSize;
+}
+
+FileNamesTable*
+UTIL_mergeFileNamesTable(FileNamesTable* table1, FileNamesTable* table2)
+{
+ unsigned newTableIdx = 0;
+ size_t pos = 0;
+ size_t newTotalTableSize;
+ char* buf;
+
+ FileNamesTable* const newTable = UTIL_assembleFileNamesTable(NULL, 0, NULL);
+ CONTROL( newTable != NULL );
+
+ newTotalTableSize = getTotalTableSize(table1) + getTotalTableSize(table2);
+
+ buf = (char*) calloc(newTotalTableSize, sizeof(*buf));
+ CONTROL ( buf != NULL );
+
+ newTable->buf = buf;
+ newTable->tableSize = table1->tableSize + table2->tableSize;
+ newTable->fileNames = (const char **) calloc(newTable->tableSize, sizeof(*(newTable->fileNames)));
+ CONTROL ( newTable->fileNames != NULL );
+
+ { unsigned idx1;
+ for( idx1=0 ; (idx1 < table1->tableSize) && table1->fileNames[idx1] && (pos < newTotalTableSize); ++idx1, ++newTableIdx) {
+ size_t const curLen = strlen(table1->fileNames[idx1]);
+ memcpy(buf+pos, table1->fileNames[idx1], curLen);
+ assert(newTableIdx <= newTable->tableSize);
+ newTable->fileNames[newTableIdx] = buf+pos;
+ pos += curLen+1;
+ } }
+
+ { unsigned idx2;
+ for( idx2=0 ; (idx2 < table2->tableSize) && table2->fileNames[idx2] && (pos < newTotalTableSize) ; ++idx2, ++newTableIdx) {
+ size_t const curLen = strlen(table2->fileNames[idx2]);
+ memcpy(buf+pos, table2->fileNames[idx2], curLen);
+ assert(newTableIdx <= newTable->tableSize);
+ newTable->fileNames[newTableIdx] = buf+pos;
+ pos += curLen+1;
+ } }
+ assert(pos <= newTotalTableSize);
+ newTable->tableSize = newTableIdx;
+
+ UTIL_freeFileNamesTable(table1);
+ UTIL_freeFileNamesTable(table2);
+
+ return newTable;
}
#ifdef _WIN32
-int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
+static int UTIL_prepareFileList(const char* dirName,
+ char** bufStart, size_t* pos,
+ char** bufEnd, int followLinks)
{
char* path;
- int dirLength, fnameLength, pathLength, nbFiles = 0;
+ size_t dirLength, pathLength;
+ int nbFiles = 0;
WIN32_FIND_DATAA cFile;
HANDLE hFile;
- dirLength = (int)strlen(dirName);
+ dirLength = strlen(dirName);
path = (char*) malloc(dirLength + 3);
if (!path) return 0;
@@ -214,7 +495,7 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
free(path);
do {
- fnameLength = (int)strlen(cFile.cFileName);
+ size_t const fnameLength = strlen(cFile.cFileName);
path = (char*) malloc(dirLength + fnameLength + 2);
if (!path) { FindClose(hFile); return 0; }
memcpy(path, dirName, dirLength);
@@ -242,8 +523,7 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
memcpy(*bufStart + *pos, path, pathLength+1 /* include final \0 */);
*pos += pathLength + 1;
nbFiles++;
- }
- }
+ } }
free(path);
} while (FindNextFileA(hFile, &cFile));
@@ -253,12 +533,13 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */
-int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
+static int UTIL_prepareFileList(const char *dirName,
+ char** bufStart, size_t* pos,
+ char** bufEnd, int followLinks)
{
- DIR *dir;
- struct dirent *entry;
- char* path;
- size_t dirLength, fnameLength, pathLength;
+ DIR* dir;
+ struct dirent * entry;
+ size_t dirLength;
int nbFiles = 0;
if (!(dir = opendir(dirName))) {
@@ -269,6 +550,8 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
dirLength = strlen(dirName);
errno = 0;
while ((entry = readdir(dir)) != NULL) {
+ char* path;
+ size_t fnameLength, pathLength;
if (strcmp (entry->d_name, "..") == 0 ||
strcmp (entry->d_name, ".") == 0) continue;
fnameLength = strlen(entry->d_name);
@@ -302,14 +585,13 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
memcpy(*bufStart + *pos, path, pathLength + 1); /* with final \0 */
*pos += pathLength + 1;
nbFiles++;
- }
- }
+ } }
free(path);
errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */
}
if (errno != 0) {
- UTIL_DISPLAYLEVEL(1, "readdir(%s) error: %s\n", dirName, strerror(errno));
+ UTIL_DISPLAYLEVEL(1, "readdir(%s) error: %s \n", dirName, strerror(errno));
free(*bufStart);
*bufStart = NULL;
}
@@ -319,10 +601,12 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
#else
-int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
+static int UTIL_prepareFileList(const char *dirName,
+ char** bufStart, size_t* pos,
+ char** bufEnd, int followLinks)
{
(void)bufStart; (void)bufEnd; (void)pos; (void)followLinks;
- UTIL_DISPLAYLEVEL(1, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName);
+ UTIL_DISPLAYLEVEL(1, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE) \n", dirName);
return 0;
}
@@ -349,68 +633,70 @@ const char* UTIL_getFileExtension(const char* infilename)
return extension;
}
-/*
- * UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories,
- * and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb).
- * After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer)
- * In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called.
- */
-const char**
-UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
- char** allocatedBuffer, unsigned* allocatedNamesNb,
- int followLinks)
+
+FileNamesTable*
+UTIL_createExpandedFNT(const char** inputNames, size_t nbIfns, int followLinks)
{
- size_t pos;
- unsigned i, nbFiles;
+ unsigned nbFiles;
char* buf = (char*)malloc(LIST_SIZE_INCREASE);
char* bufend = buf + LIST_SIZE_INCREASE;
if (!buf) return NULL;
- for (i=0, pos=0, nbFiles=0; i<inputNamesNb; i++) {
- if (!UTIL_isDirectory(inputNames[i])) {
- size_t const len = strlen(inputNames[i]);
- if (buf + pos + len >= bufend) {
- ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
- assert(newListSize >= 0);
- buf = (char*)UTIL_realloc(buf, (size_t)newListSize);
- bufend = buf + newListSize;
- if (!buf) return NULL;
- }
- if (buf + pos + len < bufend) {
- memcpy(buf+pos, inputNames[i], len+1); /* including final \0 */
- pos += len + 1;
- nbFiles++;
- }
- } else {
- nbFiles += (unsigned)UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks);
- if (buf == NULL) return NULL;
- } }
+ { size_t ifnNb, pos;
+ for (ifnNb=0, pos=0, nbFiles=0; ifnNb<nbIfns; ifnNb++) {
+ if (!UTIL_isDirectory(inputNames[ifnNb])) {
+ size_t const len = strlen(inputNames[ifnNb]);
+ if (buf + pos + len >= bufend) {
+ ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
+ assert(newListSize >= 0);
+ buf = (char*)UTIL_realloc(buf, (size_t)newListSize);
+ if (!buf) return NULL;
+ bufend = buf + newListSize;
+ }
+ if (buf + pos + len < bufend) {
+ memcpy(buf+pos, inputNames[ifnNb], len+1); /* including final \0 */
+ pos += len + 1;
+ nbFiles++;
+ }
+ } else {
+ nbFiles += (unsigned)UTIL_prepareFileList(inputNames[ifnNb], &buf, &pos, &bufend, followLinks);
+ if (buf == NULL) return NULL;
+ } } }
- if (nbFiles == 0) { free(buf); return NULL; }
+ /* note : even if nbFiles==0, function returns a valid, though empty, FileNamesTable* object */
- { const char** const fileTable = (const char**)malloc((nbFiles + 1) * sizeof(*fileTable));
- if (!fileTable) { free(buf); return NULL; }
+ { size_t ifnNb, pos;
+ size_t const fntCapacity = nbFiles + 1; /* minimum 1, allows adding one reference, typically stdin */
+ const char** const fileNamesTable = (const char**)malloc(fntCapacity * sizeof(*fileNamesTable));
+ if (!fileNamesTable) { free(buf); return NULL; }
- for (i = 0, pos = 0; i < nbFiles; i++) {
- fileTable[i] = buf + pos;
- if (buf + pos > bufend) { free(buf); free((void*)fileTable); return NULL; }
- pos += strlen(fileTable[i]) + 1;
+ for (ifnNb = 0, pos = 0; ifnNb < nbFiles; ifnNb++) {
+ fileNamesTable[ifnNb] = buf + pos;
+ if (buf + pos > bufend) { free(buf); free((void*)fileNamesTable); return NULL; }
+ pos += strlen(fileNamesTable[ifnNb]) + 1;
}
-
- *allocatedBuffer = buf;
- *allocatedNamesNb = nbFiles;
-
- return fileTable;
+ return UTIL_assembleFileNamesTable2(fileNamesTable, nbFiles, fntCapacity, buf);
}
}
-/*-****************************************
-* Console log
-******************************************/
-int g_utilDisplayLevel;
+void UTIL_expandFNT(FileNamesTable** fnt, int followLinks)
+{
+ FileNamesTable* const newFNT = UTIL_createExpandedFNT((*fnt)->fileNames, (*fnt)->tableSize, followLinks);
+ CONTROL(newFNT != NULL);
+ UTIL_freeFileNamesTable(*fnt);
+ *fnt = newFNT;
+}
+FileNamesTable* UTIL_createFNT_fromROTable(const char** filenames, size_t nbFilenames)
+{
+ size_t const sizeof_FNTable = nbFilenames * sizeof(*filenames);
+ const char** const newFNTable = (const char**)malloc(sizeof_FNTable);
+ if (newFNTable==NULL) return NULL;
+ memcpy((void*)newFNTable, filenames, sizeof_FNTable); /* void* : mitigate a Visual compiler bug or limitation */
+ return UTIL_assembleFileNamesTable(newFNTable, nbFilenames, NULL);
+}
/*-****************************************
@@ -465,8 +751,7 @@ int UTIL_countPhysicalCores(void)
}
} else {
done = TRUE;
- }
- }
+ } }
ptr = buffer;
@@ -578,8 +863,7 @@ int UTIL_countPhysicalCores(void)
} else if (ferror(cpuinfo)) {
/* fall back on the sysconf value */
goto failed;
- }
- }
+ } }
if (siblings && cpu_cores) {
ratio = siblings / cpu_cores;
}
@@ -621,7 +905,7 @@ int UTIL_countPhysicalCores(void)
return numPhysicalCores;
}
-#elif defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)
+#elif defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__) || defined(__CYGWIN__)
/* Use POSIX sysconf
* see: man 3 sysconf */
diff --git a/programs/util.h b/programs/util.h
index 1f524f2934ad..8e187e4f2999 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2020, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -20,37 +20,23 @@ extern "C" {
* Dependencies
******************************************/
#include "platform.h" /* PLATFORM_POSIX_VERSION, ZSTD_NANOSLEEP_SUPPORT, ZSTD_SETPRIORITY_SUPPORT */
-#include <stdlib.h> /* malloc, realloc, free */
#include <stddef.h> /* size_t, ptrdiff_t */
-#include <stdio.h> /* fprintf */
#include <sys/types.h> /* stat, utime */
#include <sys/stat.h> /* stat, chmod */
-#if defined(_WIN32)
-# include <sys/utime.h> /* utime */
-# include <io.h> /* _chmod */
-#else
-# include <unistd.h> /* chown, stat */
-#if PLATFORM_POSIX_VERSION < 200809L
-# include <utime.h> /* utime */
-#else
-# include <fcntl.h> /* AT_FDCWD */
-# include <sys/stat.h> /* utimensat */
-#endif
-#endif
-#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC, nanosleep */
-#include "mem.h" /* U32, U64 */
+#include "../lib/common/mem.h" /* U64 */
+
/*-************************************************************
* Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW
***************************************************************/
#if defined(_MSC_VER) && (_MSC_VER >= 1400)
-# define UTIL_fseek _fseeki64
+# define UTIL_fseek _fseeki64
#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */
# define UTIL_fseek fseeko
#elif defined(__MINGW32__) && defined(__MSVCRT__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS)
-# define UTIL_fseek fseeko64
+# define UTIL_fseek fseeko64
#else
-# define UTIL_fseek fseek
+# define UTIL_fseek fseek
#endif
@@ -85,12 +71,6 @@ extern "C" {
#endif
-/*-*************************************
-* Constants
-***************************************/
-#define LIST_SIZE_INCREASE (8*1024)
-
-
/*-****************************************
* Compiler specifics
******************************************/
@@ -112,16 +92,14 @@ extern "C" {
* Console log
******************************************/
extern int g_utilDisplayLevel;
-#define UTIL_DISPLAY(...) fprintf(stderr, __VA_ARGS__)
-#define UTIL_DISPLAYLEVEL(l, ...) { if (g_utilDisplayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } }
/*-****************************************
* File functions
******************************************/
#if defined(_MSC_VER)
- #define chmod _chmod
typedef struct __stat64 stat_t;
+ typedef int mode_t;
#else
typedef struct stat stat_t;
#endif
@@ -129,64 +107,127 @@ extern int g_utilDisplayLevel;
int UTIL_fileExist(const char* filename);
int UTIL_isRegularFile(const char* infilename);
-int UTIL_setFileStat(const char* filename, stat_t* statbuf);
-U32 UTIL_isDirectory(const char* infilename);
-int UTIL_getFileStat(const char* infilename, stat_t* statbuf);
+int UTIL_isDirectory(const char* infilename);
int UTIL_isSameFile(const char* file1, const char* file2);
-int UTIL_compareStr(const void *p1, const void *p2);
int UTIL_isCompressedFile(const char* infilename, const char *extensionList[]);
-const char* UTIL_getFileExtension(const char* infilename);
+int UTIL_isLink(const char* infilename);
+int UTIL_isFIFO(const char* infilename);
-#ifndef _MSC_VER
-U32 UTIL_isFIFO(const char* infilename);
-#endif
-U32 UTIL_isLink(const char* infilename);
#define UTIL_FILESIZE_UNKNOWN ((U64)(-1))
U64 UTIL_getFileSize(const char* infilename);
+U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles);
+int UTIL_getFileStat(const char* infilename, stat_t* statbuf);
+int UTIL_setFileStat(const char* filename, stat_t* statbuf);
+int UTIL_chmod(char const* filename, mode_t permissions); /*< like chmod, but avoid changing permission of /dev/null */
+int UTIL_compareStr(const void *p1, const void *p2);
+const char* UTIL_getFileExtension(const char* infilename);
-U64 UTIL_getTotalFileSize(const char* const * const fileNamesTable, unsigned nbFiles);
-/*
- * A modified version of realloc().
- * If UTIL_realloc() fails the original block is freed.
-*/
-UTIL_STATIC void* UTIL_realloc(void *ptr, size_t size)
-{
- void *newptr = realloc(ptr, size);
- if (newptr) return newptr;
- free(ptr);
- return NULL;
-}
+/*-****************************************
+ * Lists of Filenames
+ ******************************************/
+
+typedef struct
+{ const char** fileNames;
+ char* buf; /* fileNames are stored in this buffer (or are read-only) */
+ size_t tableSize; /* nb of fileNames */
+ size_t tableCapacity;
+} FileNamesTable;
+
+/*! UTIL_createFileNamesTable_fromFileName() :
+ * read filenames from @inputFileName, and store them into returned object.
+ * @return : a FileNamesTable*, or NULL in case of error (ex: @inputFileName doesn't exist).
+ * Note: inputFileSize must be less than 50MB
+ */
+FileNamesTable*
+UTIL_createFileNamesTable_fromFileName(const char* inputFileName);
+
+/*! UTIL_assembleFileNamesTable() :
+ * This function takes ownership of its arguments, @filenames and @buf,
+ * and store them inside the created object.
+ * note : this function never fails,
+ * it will rather exit() the program if internal allocation fails.
+ * @return : resulting FileNamesTable* object.
+ */
+FileNamesTable*
+UTIL_assembleFileNamesTable(const char** filenames, size_t tableSize, char* buf);
-int UTIL_prepareFileList(const char* dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks);
+/*! UTIL_freeFileNamesTable() :
+ * This function is compatible with NULL argument and never fails.
+ */
+void UTIL_freeFileNamesTable(FileNamesTable* table);
+
+/*! UTIL_mergeFileNamesTable():
+ * @return : FileNamesTable*, concatenation of @table1 and @table2
+ * note: @table1 and @table2 are consumed (freed) by this operation
+ */
+FileNamesTable*
+UTIL_mergeFileNamesTable(FileNamesTable* table1, FileNamesTable* table2);
+
+
+/*! UTIL_expandFNT() :
+ * read names from @fnt, and expand those corresponding to directories
+ * update @fnt, now containing only file names,
+ * @return : 0 in case of success, 1 if error
+ * note : in case of error, @fnt[0] is NULL
+ */
+void UTIL_expandFNT(FileNamesTable** fnt, int followLinks);
+
+/*! UTIL_createFNT_fromROTable() :
+ * copy the @filenames pointer table inside the returned object.
+ * The names themselves are still stored in their original buffer, which must outlive the object.
+ * @return : a FileNamesTable* object,
+ * or NULL in case of error
+ */
+FileNamesTable*
+UTIL_createFNT_fromROTable(const char** filenames, size_t nbFilenames);
+
+/*! UTIL_allocateFileNamesTable() :
+ * Allocates a table of const char*, to insert read-only names later on.
+ * The created FileNamesTable* doesn't hold a buffer.
+ * @return : FileNamesTable*, or NULL, if allocation fails.
+ */
+FileNamesTable* UTIL_allocateFileNamesTable(size_t tableSize);
+
+
+/*! UTIL_refFilename() :
+ * Add a reference to read-only name into @fnt table.
+ * As @filename is only referenced, its lifetime must outlive @fnt.
+ * Internal table must be large enough to reference a new member,
+ * otherwise its UB (protected by an `assert()`).
+ */
+void UTIL_refFilename(FileNamesTable* fnt, const char* filename);
+
+
+/* UTIL_createExpandedFNT() is only active if UTIL_HAS_CREATEFILELIST is defined.
+ * Otherwise, UTIL_createExpandedFNT() is a shell function which does nothing
+ * apart from displaying a warning message.
+ */
#ifdef _WIN32
# define UTIL_HAS_CREATEFILELIST
#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */
# define UTIL_HAS_CREATEFILELIST
-# include <dirent.h> /* opendir, readdir */
-# include <string.h> /* strerror, memcpy */
#else
-#endif /* #ifdef _WIN32 */
+ /* do not define UTIL_HAS_CREATEFILELIST */
+#endif
-/*
- * UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories,
- * and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb).
- * After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer)
- * In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called.
+/*! UTIL_createExpandedFNT() :
+ * read names from @filenames, and expand those corresponding to directories.
+ * links are followed or not depending on @followLinks directive.
+ * @return : an expanded FileNamesTable*, where each name is a file
+ * or NULL in case of error
*/
-const char**
-UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
- char** allocatedBuffer, unsigned* allocatedNamesNb,
- int followLinks);
-
-UTIL_STATIC void UTIL_freeFileList(const char** filenameTable, char* allocatedBuffer)
-{
- if (allocatedBuffer) free(allocatedBuffer);
- if (filenameTable) free((void*)filenameTable);
-}
+FileNamesTable*
+UTIL_createExpandedFNT(const char** filenames, size_t nbFilenames, int followLinks);
+
+
+/*-****************************************
+ * System
+ ******************************************/
int UTIL_countPhysicalCores(void);
+
#if defined (__cplusplus)
}
#endif
diff --git a/programs/zstd.1 b/programs/zstd.1
index fef0e76e081a..9ba0b4fa4162 100644
--- a/programs/zstd.1
+++ b/programs/zstd.1
@@ -1,5 +1,5 @@
.
-.TH "ZSTD" "1" "October 2019" "zstd 1.4.4" "User Commands"
+.TH "ZSTD" "1" "May 2020" "zstd 1.4.5" "User Commands"
.
.SH "NAME"
\fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
@@ -95,120 +95,115 @@ Display information related to a zstd compressed file, such as size, ratio, and
.
.SS "Operation modifiers"
.
-.TP
-\fB\-#\fR
-\fB#\fR compression level [1\-19] (default: 3)
+.IP "\(bu" 4
+\fB\-#\fR: \fB#\fR compression level [1\-19] (default: 3)
.
-.TP
-\fB\-\-fast[=#]\fR
-switch to ultra\-fast compression levels\. If \fB=#\fR is not present, it defaults to \fB1\fR\. The higher the value, the faster the compression speed, at the cost of some compression ratio\. This setting overwrites compression level if one was set previously\. Similarly, if a compression level is set after \fB\-\-fast\fR, it overrides it\.
+.IP "\(bu" 4
+\fB\-\-fast[=#]\fR: switch to ultra\-fast compression levels\. If \fB=#\fR is not present, it defaults to \fB1\fR\. The higher the value, the faster the compression speed, at the cost of some compression ratio\. This setting overwrites compression level if one was set previously\. Similarly, if a compression level is set after \fB\-\-fast\fR, it overrides it\.
.
-.TP
-\fB\-\-ultra\fR
-unlocks high compression levels 20+ (maximum 22), using a lot more memory\. Note that decompression will also require more memory when using these levels\.
+.IP "\(bu" 4
+\fB\-\-ultra\fR: unlocks high compression levels 20+ (maximum 22), using a lot more memory\. Note that decompression will also require more memory when using these levels\.
.
-.TP
-\fB\-\-long[=#]\fR
-enables long distance matching with \fB#\fR \fBwindowLog\fR, if not \fB#\fR is not present it defaults to \fB27\fR\. This increases the window size (\fBwindowLog\fR) and memory usage for both the compressor and decompressor\. This setting is designed to improve the compression ratio for files with long matches at a large distance\.
+.IP "\(bu" 4
+\fB\-\-long[=#]\fR: enables long distance matching with \fB#\fR \fBwindowLog\fR, if not \fB#\fR is not present it defaults to \fB27\fR\. This increases the window size (\fBwindowLog\fR) and memory usage for both the compressor and decompressor\. This setting is designed to improve the compression ratio for files with long matches at a large distance\.
.
.IP
Note: If \fBwindowLog\fR is set to larger than 27, \fB\-\-long=windowLog\fR or \fB\-\-memory=windowSize\fR needs to be passed to the decompressor\.
.
-.TP
-\fB\-T#\fR, \fB\-\-threads=#\fR
-Compress using \fB#\fR working threads (default: 1)\. If \fB#\fR is 0, attempt to detect and use the number of physical CPU cores\. In all cases, the nb of threads is capped to ZSTDMT_NBTHREADS_MAX==200\. This modifier does nothing if \fBzstd\fR is compiled without multithread support\.
+.IP "\(bu" 4
+\fB\-\-patch\-from=FILE\fR: Specify the file to be used as a reference point for zstd\'s diff engine\. This is effectively dictionary compression with some convenient parameter selection, namely that windowSize > srcSize\.
.
-.TP
-\fB\-\-single\-thread\fR
-Does not spawn a thread for compression, use a single thread for both I/O and compression\. In this mode, compression is serialized with I/O, which is slightly slower\. (This is different from \fB\-T1\fR, which spawns 1 compression thread in parallel of I/O)\. This mode is the only one available when multithread support is disabled\. Single\-thread mode features lower memory usage\. Final compressed result is slightly different from \fB\-T1\fR\.
+.IP
+Note: cannot use both this and \-D together Note: \fB\-\-long\fR mode will be automatically activated if chainLog < fileLog (fileLog being the windowLog requried to cover the whole file)\. You can also manually force it\. Node: for all levels, you can use \-\-patch\-from in \-\-single\-thread mode to improve compression ratio at the cost of speed Note: for level 19, you can get increased compression ratio at the cost of speed by specifying \fB\-\-zstd=targetLength=\fR to be something large (i\.e 4096), and by setting a large \fB\-\-zstd=chainLog=\fR
.
-.TP
-\fB\-\-adapt[=min=#,max=#]\fR
-\fBzstd\fR will dynamically adapt compression level to perceived I/O conditions\. Compression level adaptation can be observed live by using command \fB\-v\fR\. Adaptation can be constrained between supplied \fBmin\fR and \fBmax\fR levels\. The feature works when combined with multi\-threading and \fB\-\-long\fR mode\. It does not work with \fB\-\-single\-thread\fR\. It sets window size to 8 MB by default (can be changed manually, see \fBwlog\fR)\. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible\. \fInote\fR : at the time of this writing, \fB\-\-adapt\fR can remain stuck at low speed when combined with multiple worker threads (>=2)\.
+.IP "\(bu" 4
+\fB\-M#\fR, \fB\-\-memory=#\fR: Set a memory usage limit\. By default, Zstandard uses 128 MB for decompression as the maximum amount of memory the decompressor is allowed to use, but you can override this manually if need be in either direction (ie\. you can increase or decrease it)\.
.
-.TP
-\fB\-\-stream\-size=#\fR
-Sets the pledged source size of input coming from a stream\. This value must be exact, as it will be included in the produced frame header\. Incorrect stream sizes will cause an error\. This information will be used to better optimize compression parameters, resulting in better and potentially faster compression, especially for smaller source sizes\.
+.IP
+This is also used during compression when using with \-\-patch\-from=\. In this case, this parameter overrides that maximum size allowed for a dictionary\. (128 MB)\.
.
-.TP
-\fB\-\-size\-hint=#\fR
-When handling input from a stream, \fBzstd\fR must guess how large the source size will be when optimizing compression parameters\. If the stream size is relatively small, this guess may be a poor one, resulting in a higher compression ratio than expected\. This feature allows for controlling the guess when needed\. Exact guesses result in better compression ratios\. Overestimates result in slightly degraded compression ratios, while underestimates may result in significant degradation\.
+.IP "\(bu" 4
+\fB\-T#\fR, \fB\-\-threads=#\fR: Compress using \fB#\fR working threads (default: 1)\. If \fB#\fR is 0, attempt to detect and use the number of physical CPU cores\. In all cases, the nb of threads is capped to ZSTDMT_NBTHREADS_MAX==200\. This modifier does nothing if \fBzstd\fR is compiled without multithread support\.
.
-.TP
-\fB\-\-rsyncable\fR
-\fBzstd\fR will periodically synchronize the compression state to make the compressed file more rsync\-friendly\. There is a negligible impact to compression ratio, and the faster compression levels will see a small compression speed hit\. This feature does not work with \fB\-\-single\-thread\fR\. You probably don\'t want to use it with long range mode, since it will decrease the effectiveness of the synchronization points, but your milage may vary\.
+.IP "\(bu" 4
+\fB\-\-single\-thread\fR: Does not spawn a thread for compression, use a single thread for both I/O and compression\. In this mode, compression is serialized with I/O, which is slightly slower\. (This is different from \fB\-T1\fR, which spawns 1 compression thread in parallel of I/O)\. This mode is the only one available when multithread support is disabled\. Single\-thread mode features lower memory usage\. Final compressed result is slightly different from \fB\-T1\fR\.
.
-.TP
-\fB\-D file\fR
-use \fBfile\fR as Dictionary to compress or decompress FILE(s)
+.IP "\(bu" 4
+\fB\-\-adapt[=min=#,max=#]\fR : \fBzstd\fR will dynamically adapt compression level to perceived I/O conditions\. Compression level adaptation can be observed live by using command \fB\-v\fR\. Adaptation can be constrained between supplied \fBmin\fR and \fBmax\fR levels\. The feature works when combined with multi\-threading and \fB\-\-long\fR mode\. It does not work with \fB\-\-single\-thread\fR\. It sets window size to 8 MB by default (can be changed manually, see \fBwlog\fR)\. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible\. \fInote\fR : at the time of this writing, \fB\-\-adapt\fR can remain stuck at low speed when combined with multiple worker threads (>=2)\.
.
-.TP
-\fB\-\-no\-dictID\fR
-do not store dictionary ID within frame header (dictionary compression)\. The decoder will have to rely on implicit knowledge about which dictionary to use, it won\'t be able to check if it\'s correct\.
+.IP "\(bu" 4
+\fB\-\-stream\-size=#\fR : Sets the pledged source size of input coming from a stream\. This value must be exact, as it will be included in the produced frame header\. Incorrect stream sizes will cause an error\. This information will be used to better optimize compression parameters, resulting in better and potentially faster compression, especially for smaller source sizes\.
.
-.TP
-\fB\-o file\fR
-save result into \fBfile\fR (only possible with a single \fIINPUT\-FILE\fR)
+.IP "\(bu" 4
+\fB\-\-size\-hint=#\fR: When handling input from a stream, \fBzstd\fR must guess how large the source size will be when optimizing compression parameters\. If the stream size is relatively small, this guess may be a poor one, resulting in a higher compression ratio than expected\. This feature allows for controlling the guess when needed\. Exact guesses result in better compression ratios\. Overestimates result in slightly degraded compression ratios, while underestimates may result in significant degradation\.
.
-.TP
-\fB\-f\fR, \fB\-\-force\fR
-overwrite output without prompting, and (de)compress symbolic links
+.IP "\(bu" 4
+\fB\-\-rsyncable\fR : \fBzstd\fR will periodically synchronize the compression state to make the compressed file more rsync\-friendly\. There is a negligible impact to compression ratio, and the faster compression levels will see a small compression speed hit\. This feature does not work with \fB\-\-single\-thread\fR\. You probably don\'t want to use it with long range mode, since it will decrease the effectiveness of the synchronization points, but your milage may vary\.
.
-.TP
-\fB\-c\fR, \fB\-\-stdout\fR
-force write to standard output, even if it is the console
+.IP "\(bu" 4
+\fB\-D file\fR: use \fBfile\fR as Dictionary to compress or decompress FILE(s)
.
-.TP
-\fB\-\-[no\-]sparse\fR
-enable / disable sparse FS support, to make files with many zeroes smaller on disk\. Creating sparse files may save disk space and speed up decompression by reducing the amount of disk I/O\. default: enabled when output is into a file, and disabled when output is stdout\. This setting overrides default and can force sparse mode over stdout\.
+.IP "\(bu" 4
+\fB\-\-no\-dictID\fR: do not store dictionary ID within frame header (dictionary compression)\. The decoder will have to rely on implicit knowledge about which dictionary to use, it won\'t be able to check if it\'s correct\.
.
-.TP
-\fB\-\-rm\fR
-remove source file(s) after successful compression or decompression
+.IP "\(bu" 4
+\fB\-o file\fR: save result into \fBfile\fR (only possible with a single \fIINPUT\-FILE\fR)
.
-.TP
-\fB\-k\fR, \fB\-\-keep\fR
-keep source file(s) after successful compression or decompression\. This is the default behavior\.
+.IP "\(bu" 4
+\fB\-f\fR, \fB\-\-force\fR: overwrite output without prompting, and (de)compress symbolic links
.
-.TP
-\fB\-r\fR
-operate recursively on directories
+.IP "\(bu" 4
+\fB\-c\fR, \fB\-\-stdout\fR: force write to standard output, even if it is the console
.
-.TP
-\fB\-\-output\-dir\-flat[=dir]\fR
-resulting files are stored into target \fBdir\fR directory, instead of same directory as origin file\. Be aware that this command can introduce name collision issues, if multiple files, from different directories, end up having the same name\. Collision resolution ensures first file with a given name will be present in \fBdir\fR, while in combination with \fB\-f\fR, the last file will be present instead\.
+.IP "\(bu" 4
+\fB\-\-[no\-]sparse\fR: enable / disable sparse FS support, to make files with many zeroes smaller on disk\. Creating sparse files may save disk space and speed up decompression by reducing the amount of disk I/O\. default: enabled when output is into a file, and disabled when output is stdout\. This setting overrides default and can force sparse mode over stdout\.
.
-.TP
-\fB\-\-format=FORMAT\fR
-compress and decompress in other formats\. If compiled with support, zstd can compress to or decompress from other compression algorithm formats\. Possibly available options are \fBzstd\fR, \fBgzip\fR, \fBxz\fR, \fBlzma\fR, and \fBlz4\fR\. If no such format is provided, \fBzstd\fR is the default\.
+.IP "\(bu" 4
+\fB\-\-[no\-]content\-size\fR: enable / disable whether or not the original size of the file is placed in the header of the compressed file\. The default option is \-\-content\-size (meaning that the original size will be placed in the header)\.
.
-.TP
-\fB\-h\fR/\fB\-H\fR, \fB\-\-help\fR
-display help/long help and exit
+.IP "\(bu" 4
+\fB\-\-rm\fR: remove source file(s) after successful compression or decompression
.
-.TP
-\fB\-V\fR, \fB\-\-version\fR
-display version number and exit\. Advanced : \fB\-vV\fR also displays supported formats\. \fB\-vvV\fR also displays POSIX support\.
+.IP "\(bu" 4
+\fB\-k\fR, \fB\-\-keep\fR: keep source file(s) after successful compression or decompression\. This is the default behavior\.
.
-.TP
-\fB\-v\fR
-verbose mode
+.IP "\(bu" 4
+\fB\-r\fR: operate recursively on directories
.
-.TP
-\fB\-q\fR, \fB\-\-quiet\fR
-suppress warnings, interactivity, and notifications\. specify twice to suppress errors too\.
+.IP "\(bu" 4
+\fB\-\-filelist=FILE\fR read a list of files to process as content from \fBFILE\fR\. Format is compatible with \fBls\fR output, with one file per line\.
.
-.TP
-\fB\-\-no\-progress\fR
-do not display the progress bar, but keep all other messages\.
+.IP "\(bu" 4
+\fB\-\-output\-dir\-flat[=dir]\fR: resulting files are stored into target \fBdir\fR directory, instead of same directory as origin file\. Be aware that this command can introduce name collision issues, if multiple files, from different directories, end up having the same name\. Collision resolution ensures first file with a given name will be present in \fBdir\fR, while in combination with \fB\-f\fR, the last file will be present instead\.
.
-.TP
-\fB\-C\fR, \fB\-\-[no\-]check\fR
-add integrity check computed from uncompressed data (default: enabled)
+.IP "\(bu" 4
+\fB\-\-format=FORMAT\fR: compress and decompress in other formats\. If compiled with support, zstd can compress to or decompress from other compression algorithm formats\. Possibly available options are \fBzstd\fR, \fBgzip\fR, \fBxz\fR, \fBlzma\fR, and \fBlz4\fR\. If no such format is provided, \fBzstd\fR is the default\.
.
-.TP
-\fB\-\-\fR
-All arguments after \fB\-\-\fR are treated as files
+.IP "\(bu" 4
+\fB\-h\fR/\fB\-H\fR, \fB\-\-help\fR: display help/long help and exit
+.
+.IP "\(bu" 4
+\fB\-V\fR, \fB\-\-version\fR: display version number and exit\. Advanced : \fB\-vV\fR also displays supported formats\. \fB\-vvV\fR also displays POSIX support\.
+.
+.IP "\(bu" 4
+\fB\-v\fR, \fB\-\-verbose\fR: verbose mode
+.
+.IP "\(bu" 4
+\fB\-\-show\-default\-cparams\fR: Shows the default compresssion parameters that will be used for a particular src file\. If the provided src file is not a regular file (eg\. named pipe), the cli will just output the default paramters\. That is, the parameters that are used when the src size is unknown\.
+.
+.IP "\(bu" 4
+\fB\-q\fR, \fB\-\-quiet\fR: suppress warnings, interactivity, and notifications\. specify twice to suppress errors too\.
+.
+.IP "\(bu" 4
+\fB\-\-no\-progress\fR: do not display the progress bar, but keep all other messages\.
+.
+.IP "\(bu" 4
+\fB\-C\fR, \fB\-\-[no\-]check\fR: add integrity check computed from uncompressed data (default: enabled)
+.
+.IP "\(bu" 4
+\fB\-\-\fR: All arguments after \fB\-\-\fR are treated as files
+.
+.IP "" 0
.
.SS "Restricted usage of Environment Variables"
Using environment variables to set parameters has security implications\. Therefore, this avenue is intentionally restricted\. Only \fBZSTD_CLEVEL\fR is supported currently, for setting compression level\. \fBZSTD_CLEVEL\fR can be used to set the level between 1 and 19 (the "normal" range)\. If the value of \fBZSTD_CLEVEL\fR is not a valid integer, it will be ignored with a warning message\. \fBZSTD_CLEVEL\fR just replaces the default compression level (\fB3\fR)\. It can be overridden by corresponding command line arguments\.
@@ -361,7 +356,7 @@ Specify the maximum number of bits for a hash table\.
Bigger hash tables cause less collisions which usually makes compression faster, but requires more memory during compression\.
.
.IP
-The minimum \fIhlog\fR is 6 (64 B) and the maximum is 26 (128 MiB)\.
+The minimum \fIhlog\fR is 6 (64 B) and the maximum is 30 (1 GiB)\.
.
.TP
\fBchainLog\fR=\fIclog\fR, \fBclog\fR=\fIclog\fR
@@ -371,7 +366,7 @@ Specify the maximum number of bits for a hash chain or a binary tree\.
Higher numbers of bits increases the chance to find a match which usually improves compression ratio\. It also slows down compression speed and increases memory requirements for compression\. This option is ignored for the ZSTD_fast strategy\.
.
.IP
-The minimum \fIclog\fR is 6 (64 B) and the maximum is 28 (256 MiB)\.
+The minimum \fIclog\fR is 6 (64 B) and the maximum is 29 (524 Mib) on 32\-bit platforms and 30 (1 Gib) on 64\-bit platforms\.
.
.TP
\fBsearchLog\fR=\fIslog\fR, \fBslog\fR=\fIslog\fR
@@ -381,7 +376,7 @@ Specify the maximum number of searches in a hash chain or a binary tree using lo
More searches increases the chance to find a match which usually increases compression ratio but decreases compression speed\.
.
.IP
-The minimum \fIslog\fR is 1 and the maximum is 26\.
+The minimum \fIslog\fR is 1 and the maximum is \'windowLog\' \- 1\.
.
.TP
\fBminMatch\fR=\fImml\fR, \fBmml\fR=\fImml\fR
@@ -394,20 +389,17 @@ Larger search lengths usually decrease compression ratio but improve decompressi
The minimum \fImml\fR is 3 and the maximum is 7\.
.
.TP
-\fBtargetLen\fR=\fItlen\fR, \fBtlen\fR=\fItlen\fR
+\fBtargetLength\fR=\fItlen\fR, \fBtlen\fR=\fItlen\fR
The impact of this field vary depending on selected strategy\.
.
.IP
-For ZSTD_btopt, ZSTD_btultra and ZSTD_btultra2, it specifies the minimum match length that causes match finder to stop searching\. A larger \fBtargetLen\fR usually improves compression ratio but decreases compression speed\.
-.
-.IP
-For ZSTD_fast, it triggers ultra\-fast mode when > 0\. The value represents the amount of data skipped between match sampling\. Impact is reversed : a larger \fBtargetLen\fR increases compression speed but decreases compression ratio\.
+For ZSTD_btopt, ZSTD_btultra and ZSTD_btultra2, it specifies the minimum match length that causes match finder to stop searching\. A larger \fBtargetLength\fR usually improves compression ratio but decreases compression speed\. t For ZSTD_fast, it triggers ultra\-fast mode when > 0\. The value represents the amount of data skipped between match sampling\. Impact is reversed : a larger \fBtargetLength\fR increases compression speed but decreases compression ratio\.
.
.IP
For all other strategies, this field has no impact\.
.
.IP
-The minimum \fItlen\fR is 0 and the maximum is 999\.
+The minimum \fItlen\fR is 0 and the maximum is 128 Kib\.
.
.TP
\fBoverlapLog\fR=\fIovlog\fR, \fBovlog\fR=\fIovlog\fR
@@ -427,7 +419,7 @@ This option is ignored unless long distance matching is enabled\.
Bigger hash tables usually improve compression ratio at the expense of more memory during compression and a decrease in compression speed\.
.
.IP
-The minimum \fIlhlog\fR is 6 and the maximum is 26 (default: 20)\.
+The minimum \fIlhlog\fR is 6 and the maximum is 30 (default: 20)\.
.
.TP
\fBldmMinMatch\fR=\fIlmml\fR, \fBlmml\fR=\fIlmml\fR
@@ -453,7 +445,7 @@ This option is ignored unless long distance matching is enabled\.
Larger bucket sizes improve collision resolution but decrease compression speed\.
.
.IP
-The minimum \fIlblog\fR is 0 and the maximum is 8 (default: 3)\.
+The minimum \fIlblog\fR is 1 and the maximum is 8 (default: 3)\.
.
.TP
\fBldmHashRateLog\fR=\fIlhrlog\fR, \fBlhrlog\fR=\fIlhrlog\fR
diff --git a/programs/zstd.1.md b/programs/zstd.1.md
index e3daa4c87ac7..550e2e53fe5f 100644
--- a/programs/zstd.1.md
+++ b/programs/zstd.1.md
@@ -122,6 +122,28 @@ the last one takes effect.
Note: If `windowLog` is set to larger than 27, `--long=windowLog` or
`--memory=windowSize` needs to be passed to the decompressor.
+* `--patch-from=FILE`:
+ Specify the file to be used as a reference point for zstd's diff engine.
+ This is effectively dictionary compression with some convenient parameter
+ selection, namely that windowSize > srcSize.
+
+ Note: cannot use both this and -D together
+ Note: `--long` mode will be automatically activated if chainLog < fileLog
+ (fileLog being the windowLog requried to cover the whole file). You
+ can also manually force it.
+ Node: for all levels, you can use --patch-from in --single-thread mode
+ to improve compression ratio at the cost of speed
+ Note: for level 19, you can get increased compression ratio at the cost
+ of speed by specifying `--zstd=targetLength=` to be something large
+ (i.e 4096), and by setting a large `--zstd=chainLog=`
+* `-M#`, `--memory=#`:
+ Set a memory usage limit. By default, Zstandard uses 128 MB for decompression
+ as the maximum amount of memory the decompressor is allowed to use, but you can
+ override this manually if need be in either direction (ie. you can increase or
+ decrease it).
+
+ This is also used during compression when using with --patch-from=. In this case,
+ this parameter overrides that maximum size allowed for a dictionary. (128 MB).
* `-T#`, `--threads=#`:
Compress using `#` working threads (default: 1).
If `#` is 0, attempt to detect and use the number of physical CPU cores.
@@ -184,6 +206,10 @@ the last one takes effect.
default: enabled when output is into a file,
and disabled when output is stdout.
This setting overrides default and can force sparse mode over stdout.
+* `--[no-]content-size`:
+ enable / disable whether or not the original size of the file is placed in
+ the header of the compressed file. The default option is
+ --content-size (meaning that the original size will be placed in the header).
* `--rm`:
remove source file(s) after successful compression or decompression
* `-k`, `--keep`:
@@ -191,6 +217,9 @@ the last one takes effect.
This is the default behavior.
* `-r`:
operate recursively on directories
+* `--filelist=FILE`
+ read a list of files to process as content from `FILE`.
+ Format is compatible with `ls` output, with one file per line.
* `--output-dir-flat[=dir]`:
resulting files are stored into target `dir` directory,
instead of same directory as origin file.
@@ -209,8 +238,14 @@ the last one takes effect.
display version number and exit.
Advanced : `-vV` also displays supported formats.
`-vvV` also displays POSIX support.
-* `-v`:
+* `-v`, `--verbose`:
verbose mode
+* `--show-default-cparams`:
+ Shows the default compresssion parameters that will be used for a
+ particular src file. If the provided src file is not a regular file
+ (eg. named pipe), the cli will just output the default paramters.
+ That is, the parameters that are used when the src size is
+ unknown.
* `-q`, `--quiet`:
suppress warnings, interactivity, and notifications.
specify twice to suppress errors too.
@@ -402,7 +437,7 @@ The list of available _options_:
Bigger hash tables cause less collisions which usually makes compression
faster, but requires more memory during compression.
- The minimum _hlog_ is 6 (64 B) and the maximum is 26 (128 MiB).
+ The minimum _hlog_ is 6 (64 B) and the maximum is 30 (1 GiB).
- `chainLog`=_clog_, `clog`=_clog_:
Specify the maximum number of bits for a hash chain or a binary tree.
@@ -413,7 +448,8 @@ The list of available _options_:
compression.
This option is ignored for the ZSTD_fast strategy.
- The minimum _clog_ is 6 (64 B) and the maximum is 28 (256 MiB).
+ The minimum _clog_ is 6 (64 B) and the maximum is 29 (524 Mib) on 32-bit platforms
+ and 30 (1 Gib) on 64-bit platforms.
- `searchLog`=_slog_, `slog`=_slog_:
Specify the maximum number of searches in a hash chain or a binary tree
@@ -422,7 +458,7 @@ The list of available _options_:
More searches increases the chance to find a match which usually increases
compression ratio but decreases compression speed.
- The minimum _slog_ is 1 and the maximum is 26.
+ The minimum _slog_ is 1 and the maximum is 'windowLog' - 1.
- `minMatch`=_mml_, `mml`=_mml_:
Specify the minimum searched length of a match in a hash table.
@@ -432,22 +468,22 @@ The list of available _options_:
The minimum _mml_ is 3 and the maximum is 7.
-- `targetLen`=_tlen_, `tlen`=_tlen_:
+- `targetLength`=_tlen_, `tlen`=_tlen_:
The impact of this field vary depending on selected strategy.
For ZSTD\_btopt, ZSTD\_btultra and ZSTD\_btultra2, it specifies
the minimum match length that causes match finder to stop searching.
- A larger `targetLen` usually improves compression ratio
+ A larger `targetLength` usually improves compression ratio
but decreases compression speed.
-
+t
For ZSTD\_fast, it triggers ultra-fast mode when > 0.
The value represents the amount of data skipped between match sampling.
- Impact is reversed : a larger `targetLen` increases compression speed
+ Impact is reversed : a larger `targetLength` increases compression speed
but decreases compression ratio.
For all other strategies, this field has no impact.
- The minimum _tlen_ is 0 and the maximum is 999.
+ The minimum _tlen_ is 0 and the maximum is 128 Kib.
- `overlapLog`=_ovlog_, `ovlog`=_ovlog_:
Determine `overlapSize`, amount of data reloaded from previous job.
@@ -470,7 +506,7 @@ The list of available _options_:
Bigger hash tables usually improve compression ratio at the expense of more
memory during compression and a decrease in compression speed.
- The minimum _lhlog_ is 6 and the maximum is 26 (default: 20).
+ The minimum _lhlog_ is 6 and the maximum is 30 (default: 20).
- `ldmMinMatch`=_lmml_, `lmml`=_lmml_:
Specify the minimum searched length of a match for long distance matching.
@@ -490,7 +526,7 @@ The list of available _options_:
Larger bucket sizes improve collision resolution but decrease compression
speed.
- The minimum _lblog_ is 0 and the maximum is 8 (default: 3).
+ The minimum _lblog_ is 1 and the maximum is 8 (default: 3).
- `ldmHashRateLog`=_lhrlog_, `lhrlog`=_lhrlog_:
Specify the frequency of inserting entries into the long distance matching
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index cd6b40bc089f..70e2b70666bf 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -27,10 +27,12 @@
**************************************/
#include "platform.h" /* IS_CONSOLE, PLATFORM_POSIX_VERSION */
#include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */
-#include <stdio.h> /* fprintf(), stdin, stdout, stderr */
#include <stdlib.h> /* getenv */
#include <string.h> /* strcmp, strlen */
+#include <stdio.h> /* fprintf(), stdin, stdout, stderr */
#include <errno.h> /* errno */
+#include <assert.h> /* assert */
+
#include "fileio.h" /* stdinmark, stdoutmark, ZSTD_EXTENSION */
#ifndef ZSTD_NOBENCH
# include "benchzstd.h" /* BMK_benchFiles */
@@ -38,8 +40,7 @@
#ifndef ZSTD_NODICT
# include "dibio.h" /* ZDICT_cover_params_t, DiB_trainFromFiles() */
#endif
-#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_minCLevel */
-#include "zstd.h" /* ZSTD_VERSION_STRING, ZSTD_maxCLevel */
+#include "../lib/zstd.h" /* ZSTD_VERSION_STRING, ZSTD_minCLevel, ZSTD_maxCLevel */
/*-************************************
@@ -93,127 +94,146 @@ typedef enum { cover, fastCover, legacy } dictType;
/*-************************************
* Display Macros
**************************************/
-#define DISPLAY(...) fprintf(g_displayOut, __VA_ARGS__)
+#define DISPLAY_F(f, ...) fprintf((f), __VA_ARGS__)
+#define DISPLAYOUT(...) DISPLAY_F(stdout, __VA_ARGS__)
+#define DISPLAY(...) DISPLAY_F(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
static int g_displayLevel = DISPLAY_LEVEL_DEFAULT; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */
-static FILE* g_displayOut;
/*-************************************
* Command Line
**************************************/
-static int usage(const char* programName)
+/* print help either in `stderr` or `stdout` depending on originating request
+ * error (badusage) => stderr
+ * help (usage_advanced) => stdout
+ */
+static void usage(FILE* f, const char* programName)
{
- DISPLAY( "Usage : \n");
- DISPLAY( " %s [args] [FILE(s)] [-o file] \n", programName);
- DISPLAY( "\n");
- DISPLAY( "FILE : a filename \n");
- DISPLAY( " with no FILE, or when FILE is - , read standard input\n");
- DISPLAY( "Arguments : \n");
+ DISPLAY_F(f, "Usage : \n");
+ DISPLAY_F(f, " %s [args] [FILE(s)] [-o file] \n", programName);
+ DISPLAY_F(f, "\n");
+ DISPLAY_F(f, "FILE : a filename \n");
+ DISPLAY_F(f, " with no FILE, or when FILE is - , read standard input\n");
+ DISPLAY_F(f, "Arguments : \n");
#ifndef ZSTD_NOCOMPRESS
- DISPLAY( " -# : # compression level (1-%d, default: %d) \n", ZSTDCLI_CLEVEL_MAX, ZSTDCLI_CLEVEL_DEFAULT);
+ DISPLAY_F(f, " -# : # compression level (1-%d, default: %d) \n", ZSTDCLI_CLEVEL_MAX, ZSTDCLI_CLEVEL_DEFAULT);
#endif
#ifndef ZSTD_NODECOMPRESS
- DISPLAY( " -d : decompression \n");
+ DISPLAY_F(f, " -d : decompression \n");
#endif
- DISPLAY( " -D file: use `file` as Dictionary \n");
- DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n");
- DISPLAY( " -f : overwrite output without prompting and (de)compress links \n");
- DISPLAY( "--rm : remove source file(s) after successful de/compression \n");
- DISPLAY( " -k : preserve source file(s) (default) \n");
- DISPLAY( " -h/-H : display help/long help and exit \n");
- return 0;
+ DISPLAY_F(f, " -D DICT: use DICT as Dictionary for compression or decompression \n");
+ DISPLAY_F(f, " -o file: result stored into `file` (only 1 output file) \n");
+ DISPLAY_F(f, " -f : overwrite output without prompting, also (de)compress links \n");
+ DISPLAY_F(f, "--rm : remove source file(s) after successful de/compression \n");
+ DISPLAY_F(f, " -k : preserve source file(s) (default) \n");
+ DISPLAY_F(f, " -h/-H : display help/long help and exit \n");
}
-static int usage_advanced(const char* programName)
+static void usage_advanced(const char* programName)
{
- DISPLAY(WELCOME_MESSAGE);
- usage(programName);
- DISPLAY( "\n");
- DISPLAY( "Advanced arguments : \n");
- DISPLAY( " -V : display Version number and exit \n");
- DISPLAY( " -v : verbose mode; specify multiple times to increase verbosity\n");
- DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");
- DISPLAY( " -c : force write to standard output, even if it is the console\n");
- DISPLAY( " -l : print information about zstd compressed files \n");
- DISPLAY( "--exclude-compressed: only compress files that are not previously compressed \n");
-#ifndef ZSTD_NOCOMPRESS
- DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
- DISPLAY( "--long[=#]: enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
- DISPLAY( "--fast[=#]: switch to very fast compression levels (default: %u)\n", 1);
- DISPLAY( "--adapt : dynamically adapt compression level to I/O conditions \n");
- DISPLAY( "--stream-size=# : optimize compression parameters for streaming input of given number of bytes \n");
- DISPLAY( "--size-hint=# optimize compression parameters for streaming input of approximately this size\n");
- DISPLAY( "--target-compressed-block-size=# : make compressed block near targeted size \n");
-#ifdef ZSTD_MULTITHREAD
- DISPLAY( " -T# : spawns # compression threads (default: 1, 0==# cores) \n");
- DISPLAY( " -B# : select size of each job (default: 0==automatic) \n");
- DISPLAY( "--rsyncable : compress using a rsync-friendly method (-B sets block size) \n");
-#endif
- DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n");
- DISPLAY( "--[no-]check : integrity check (default: enabled) \n");
- DISPLAY( "--[no-]compress-literals : force (un)compressed literals \n");
-#endif
+ DISPLAYOUT(WELCOME_MESSAGE);
+ usage(stdout, programName);
+ DISPLAYOUT( "\n");
+ DISPLAYOUT( "Advanced arguments : \n");
+ DISPLAYOUT( " -V : display Version number and exit \n");
+
+ DISPLAYOUT( " -c : force write to standard output, even if it is the console \n");
+
+ DISPLAYOUT( " -v : verbose mode; specify multiple times to increase verbosity \n");
+ DISPLAYOUT( " -q : suppress warnings; specify twice to suppress errors too \n");
+ DISPLAYOUT( "--no-progress : do not display the progress counter \n");
+
#ifdef UTIL_HAS_CREATEFILELIST
- DISPLAY( " -r : operate recursively on directories \n");
- DISPLAY( "--output-dir-flat[=directory]: all resulting files stored into `directory`. \n");
+ DISPLAYOUT( " -r : operate recursively on directories \n");
+ DISPLAYOUT( "--filelist=FILE : read list of files to operate upon from FILE \n");
+ DISPLAYOUT( "--output-dir-flat=DIR : all resulting files are stored into DIR \n");
#endif
- DISPLAY( "--format=zstd : compress files to the .zst format (default) \n");
+
+ DISPLAYOUT( "-- : All arguments after \"--\" are treated as files \n");
+
+#ifndef ZSTD_NOCOMPRESS
+ DISPLAYOUT( "\n");
+ DISPLAYOUT( "Advanced compression arguments : \n");
+ DISPLAYOUT( "--ultra : enable levels beyond %i, up to %i (requires more memory) \n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
+ DISPLAYOUT( "--long[=#]: enable long distance matching with given window log (default: %u) \n", g_defaultMaxWindowLog);
+ DISPLAYOUT( "--fast[=#]: switch to very fast compression levels (default: %u) \n", 1);
+ DISPLAYOUT( "--adapt : dynamically adapt compression level to I/O conditions \n");
+# ifdef ZSTD_MULTITHREAD
+ DISPLAYOUT( " -T# : spawns # compression threads (default: 1, 0==# cores) \n");
+ DISPLAYOUT( " -B# : select size of each job (default: 0==automatic) \n");
+ DISPLAYOUT( "--single-thread : use a single thread for both I/O and compression (result slightly different than -T1) \n");
+ DISPLAYOUT( "--rsyncable : compress using a rsync-friendly method (-B sets block size) \n");
+# endif
+ DISPLAYOUT( "--exclude-compressed: only compress files that are not already compressed \n");
+ DISPLAYOUT( "--stream-size=# : specify size of streaming input from `stdin` \n");
+ DISPLAYOUT( "--size-hint=# optimize compression parameters for streaming input of approximately this size \n");
+ DISPLAYOUT( "--target-compressed-block-size=# : generate compressed block of approximately targeted size \n");
+ DISPLAYOUT( "--no-dictID : don't write dictID into header (dictionary compression only) \n");
+ DISPLAYOUT( "--[no-]check : add XXH64 integrity checksum to frame (default: enabled) \n");
+ DISPLAYOUT( "--[no-]compress-literals : force (un)compressed literals \n");
+
+ DISPLAYOUT( "--format=zstd : compress files to the .zst format (default) \n");
#ifdef ZSTD_GZCOMPRESS
- DISPLAY( "--format=gzip : compress files to the .gz format \n");
+ DISPLAYOUT( "--format=gzip : compress files to the .gz format \n");
#endif
#ifdef ZSTD_LZMACOMPRESS
- DISPLAY( "--format=xz : compress files to the .xz format \n");
- DISPLAY( "--format=lzma : compress files to the .lzma format \n");
+ DISPLAYOUT( "--format=xz : compress files to the .xz format \n");
+ DISPLAYOUT( "--format=lzma : compress files to the .lzma format \n");
#endif
#ifdef ZSTD_LZ4COMPRESS
- DISPLAY( "--format=lz4 : compress files to the .lz4 format \n");
+ DISPLAYOUT( "--format=lz4 : compress files to the .lz4 format \n");
#endif
+#endif /* !ZSTD_NOCOMPRESS */
+
#ifndef ZSTD_NODECOMPRESS
- DISPLAY( "--test : test compressed file integrity \n");
-#if ZSTD_SPARSE_DEFAULT
- DISPLAY( "--[no-]sparse : sparse mode (default: enabled on file, disabled on stdout)\n");
-#else
- DISPLAY( "--[no-]sparse : sparse mode (default: disabled)\n");
-#endif
-#endif
- DISPLAY( " -M# : Set a memory usage limit for decompression \n");
- DISPLAY( "--no-progress : do not display the progress bar \n");
- DISPLAY( "-- : All arguments after \"--\" are treated as files \n");
+ DISPLAYOUT( "\n");
+ DISPLAYOUT( "Advanced decompression arguments : \n");
+ DISPLAYOUT( " -l : print information about zstd compressed files \n");
+ DISPLAYOUT( "--test : test compressed file integrity \n");
+ DISPLAYOUT( " -M# : Set a memory usage limit for decompression \n");
+# if ZSTD_SPARSE_DEFAULT
+ DISPLAYOUT( "--[no-]sparse : sparse mode (default: enabled on file, disabled on stdout) \n");
+# else
+ DISPLAYOUT( "--[no-]sparse : sparse mode (default: disabled) \n");
+# endif
+#endif /* ZSTD_NODECOMPRESS */
+
#ifndef ZSTD_NODICT
- DISPLAY( "\n");
- DISPLAY( "Dictionary builder : \n");
- DISPLAY( "--train ## : create a dictionary from a training set of files \n");
- DISPLAY( "--train-cover[=k=#,d=#,steps=#,split=#,shrink[=#]] : use the cover algorithm with optional args\n");
- DISPLAY( "--train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#,shrink[=#]] : use the fast cover algorithm with optional args\n");
- DISPLAY( "--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: %u)\n", g_defaultSelectivityLevel);
- DISPLAY( " -o file : `file` is dictionary name (default: %s) \n", g_defaultDictName);
- DISPLAY( "--maxdict=# : limit dictionary to specified size (default: %u) \n", g_defaultMaxDictSize);
- DISPLAY( "--dictID=# : force dictionary ID to specified value (default: random)\n");
+ DISPLAYOUT( "\n");
+ DISPLAYOUT( "Dictionary builder : \n");
+ DISPLAYOUT( "--train ## : create a dictionary from a training set of files \n");
+ DISPLAYOUT( "--train-cover[=k=#,d=#,steps=#,split=#,shrink[=#]] : use the cover algorithm with optional args \n");
+ DISPLAYOUT( "--train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#,shrink[=#]] : use the fast cover algorithm with optional args \n");
+ DISPLAYOUT( "--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: %u) \n", g_defaultSelectivityLevel);
+ DISPLAYOUT( " -o DICT : DICT is dictionary name (default: %s) \n", g_defaultDictName);
+ DISPLAYOUT( "--maxdict=# : limit dictionary to specified size (default: %u) \n", g_defaultMaxDictSize);
+ DISPLAYOUT( "--dictID=# : force dictionary ID to specified value (default: random) \n");
#endif
+
#ifndef ZSTD_NOBENCH
- DISPLAY( "\n");
- DISPLAY( "Benchmark arguments : \n");
- DISPLAY( " -b# : benchmark file(s), using # compression level (default: %d) \n", ZSTDCLI_CLEVEL_DEFAULT);
- DISPLAY( " -e# : test all compression levels from -bX to # (default: 1)\n");
- DISPLAY( " -i# : minimum evaluation time in seconds (default: 3s) \n");
- DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n");
- DISPLAY( "--priority=rt : set process priority to real-time \n");
+ DISPLAYOUT( "\n");
+ DISPLAYOUT( "Benchmark arguments : \n");
+ DISPLAYOUT( " -b# : benchmark file(s), using # compression level (default: %d) \n", ZSTDCLI_CLEVEL_DEFAULT);
+ DISPLAYOUT( " -e# : test all compression levels successively from -b# to -e# (default: 1) \n");
+ DISPLAYOUT( " -i# : minimum evaluation time in seconds (default: 3s) \n");
+ DISPLAYOUT( " -B# : cut file into independent blocks of size # (default: no block) \n");
+ DISPLAYOUT( " -S : output one benchmark result per input file (default: consolidated result) \n");
+ DISPLAYOUT( "--priority=rt : set process priority to real-time \n");
#endif
- return 0;
+
}
-static int badusage(const char* programName)
+static void badusage(const char* programName)
{
- DISPLAYLEVEL(1, "Incorrect parameters\n");
- if (g_displayLevel >= 2) usage(programName);
- return 1;
+ DISPLAYLEVEL(1, "Incorrect parameters \n");
+ if (g_displayLevel >= 2) usage(stderr, programName);
}
static void waitEnter(void)
{
int unused;
- DISPLAY("Press enter to continue...\n");
+ DISPLAY("Press enter to continue... \n");
unused = getchar();
(void)unused;
}
@@ -249,10 +269,12 @@ static int readU32FromCharChecked(const char** stringPtr, unsigned* value)
{
unsigned result = 0;
while ((**stringPtr >='0') && (**stringPtr <='9')) {
- unsigned const max = (((unsigned)(-1)) / 10) - 1;
+ unsigned const max = ((unsigned)(-1)) / 10;
+ unsigned last = result;
if (result > max) return 1; /* overflow error */
result *= 10;
result += (unsigned)(**stringPtr - '0');
+ if (result < last) return 1; /* overflow error */
(*stringPtr)++ ;
}
if ((**stringPtr=='K') || (**stringPtr=='M')) {
@@ -277,12 +299,57 @@ static int readU32FromCharChecked(const char** stringPtr, unsigned* value)
* Will also modify `*stringPtr`, advancing it to position where it stopped reading.
* Note : function will exit() program if digit sequence overflows */
static unsigned readU32FromChar(const char** stringPtr) {
- static const char errorMsg[] = "error: numeric value too large";
+ static const char errorMsg[] = "error: numeric value overflows 32-bit unsigned int";
unsigned result;
if (readU32FromCharChecked(stringPtr, &result)) { errorOut(errorMsg); }
return result;
}
+/*! readSizeTFromCharChecked() :
+ * @return 0 if success, and store the result in *value.
+ * allows and interprets K, KB, KiB, M, MB and MiB suffix.
+ * Will also modify `*stringPtr`, advancing it to position where it stopped reading.
+ * @return 1 if an overflow error occurs */
+static int readSizeTFromCharChecked(const char** stringPtr, size_t* value)
+{
+ size_t result = 0;
+ while ((**stringPtr >='0') && (**stringPtr <='9')) {
+ size_t const max = ((size_t)(-1)) / 10;
+ size_t last = result;
+ if (result > max) return 1; /* overflow error */
+ result *= 10;
+ result += (size_t)(**stringPtr - '0');
+ if (result < last) return 1; /* overflow error */
+ (*stringPtr)++ ;
+ }
+ if ((**stringPtr=='K') || (**stringPtr=='M')) {
+ size_t const maxK = ((size_t)(-1)) >> 10;
+ if (result > maxK) return 1; /* overflow error */
+ result <<= 10;
+ if (**stringPtr=='M') {
+ if (result > maxK) return 1; /* overflow error */
+ result <<= 10;
+ }
+ (*stringPtr)++; /* skip `K` or `M` */
+ if (**stringPtr=='i') (*stringPtr)++;
+ if (**stringPtr=='B') (*stringPtr)++;
+ }
+ *value = result;
+ return 0;
+}
+
+/*! readSizeTFromChar() :
+ * @return : size_t value read from input in `char` format.
+ * allows and interprets K, KB, KiB, M, MB and MiB suffix.
+ * Will also modify `*stringPtr`, advancing it to position where it stopped reading.
+ * Note : function will exit() program if digit sequence overflows */
+static size_t readSizeTFromChar(const char** stringPtr) {
+ static const char errorMsg[] = "error: numeric value overflows size_t";
+ size_t result;
+ if (readSizeTFromCharChecked(stringPtr, &result)) { errorOut(errorMsg); }
+ return result;
+}
+
/** longCommandWArg() :
* check if *stringPtr is the same as longCommand.
* If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
@@ -477,42 +544,45 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi
static void printVersion(void)
{
- DISPLAY(WELCOME_MESSAGE);
+ DISPLAYOUT(WELCOME_MESSAGE);
+ if (g_displayLevel >= 3) {
/* format support */
- DISPLAYLEVEL(3, "*** supports: zstd");
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>0) && (ZSTD_LEGACY_SUPPORT<8)
- DISPLAYLEVEL(3, ", zstd legacy v0.%d+", ZSTD_LEGACY_SUPPORT);
-#endif
-#ifdef ZSTD_GZCOMPRESS
- DISPLAYLEVEL(3, ", gzip");
-#endif
-#ifdef ZSTD_LZ4COMPRESS
- DISPLAYLEVEL(3, ", lz4");
-#endif
-#ifdef ZSTD_LZMACOMPRESS
- DISPLAYLEVEL(3, ", lzma, xz ");
-#endif
- DISPLAYLEVEL(3, "\n");
- /* posix support */
-#ifdef _POSIX_C_SOURCE
- DISPLAYLEVEL(4, "_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE);
-#endif
-#ifdef _POSIX_VERSION
- DISPLAYLEVEL(4, "_POSIX_VERSION defined: %ldL \n", (long) _POSIX_VERSION);
-#endif
-#ifdef PLATFORM_POSIX_VERSION
- DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION);
-#endif
+ DISPLAYOUT("*** supports: zstd");
+ #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>0) && (ZSTD_LEGACY_SUPPORT<8)
+ DISPLAYOUT(", zstd legacy v0.%d+", ZSTD_LEGACY_SUPPORT);
+ #endif
+ #ifdef ZSTD_GZCOMPRESS
+ DISPLAYOUT(", gzip");
+ #endif
+ #ifdef ZSTD_LZ4COMPRESS
+ DISPLAYOUT(", lz4");
+ #endif
+ #ifdef ZSTD_LZMACOMPRESS
+ DISPLAYOUT(", lzma, xz ");
+ #endif
+ DISPLAYOUT("\n");
+ if (g_displayLevel >= 4) {
+ /* posix support */
+ #ifdef _POSIX_C_SOURCE
+ DISPLAYOUT("_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE);
+ #endif
+ #ifdef _POSIX_VERSION
+ DISPLAYOUT("_POSIX_VERSION defined: %ldL \n", (long) _POSIX_VERSION);
+ #endif
+ #ifdef PLATFORM_POSIX_VERSION
+ DISPLAYOUT("PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION);
+ #endif
+ } }
}
/* Environment variables for parameter setting */
#define ENV_CLEVEL "ZSTD_CLEVEL"
-/* functions that pick up environment variables */
+/* pick up environment variable */
static int init_cLevel(void) {
const char* const env = getenv(ENV_CLEVEL);
- if (env) {
- const char *ptr = env;
+ if (env != NULL) {
+ const char* ptr = env;
int sign = 1;
if (*ptr == '-') {
sign = -1;
@@ -524,33 +594,38 @@ static int init_cLevel(void) {
if ((*ptr>='0') && (*ptr<='9')) {
unsigned absLevel;
if (readU32FromCharChecked(&ptr, &absLevel)) {
- DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large\n", ENV_CLEVEL, env);
+ DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large \n", ENV_CLEVEL, env);
return ZSTDCLI_CLEVEL_DEFAULT;
} else if (*ptr == 0) {
return sign * (int)absLevel;
- }
- }
+ } }
- DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: not a valid integer value\n", ENV_CLEVEL, env);
+ DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: not a valid integer value \n", ENV_CLEVEL, env);
}
return ZSTDCLI_CLEVEL_DEFAULT;
}
+#define ZSTD_NB_STRATEGIES 9
+
+static const char* ZSTD_strategyMap[ZSTD_NB_STRATEGIES + 1] = { "", "ZSTD_fast",
+ "ZSTD_dfast", "ZSTD_greedy", "ZSTD_lazy", "ZSTD_lazy2", "ZSTD_btlazy2",
+ "ZSTD_btopt", "ZSTD_btultra", "ZSTD_btultra2"};
+
typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode;
#define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
#ifdef ZSTD_NOCOMPRESS
/* symbols from compression library are not defined and should not be invoked */
-# define MINCLEVEL -50
+# define MINCLEVEL -99
# define MAXCLEVEL 22
#else
# define MINCLEVEL ZSTD_minCLevel()
# define MAXCLEVEL ZSTD_maxCLevel()
#endif
-int main(int argCount, const char* argv[])
+int main(int const argCount, const char* argv[])
{
int argNb,
followLinks = 0,
@@ -573,7 +648,9 @@ int main(int argCount, const char* argv[])
separateFiles = 0,
setRealTimePrio = 0,
singleThread = 0,
- ultra=0;
+ showDefaultCParams = 0,
+ ultra=0,
+ contentSize=1;
double compressibility = 0.5;
unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
size_t blockSize = 0;
@@ -581,16 +658,17 @@ int main(int argCount, const char* argv[])
FIO_prefs_t* const prefs = FIO_createPreferences();
zstd_operation_mode operation = zom_compress;
ZSTD_compressionParameters compressionParams;
- int cLevel;
- int cLevelLast = -1000000000;
+ int cLevel = init_cLevel();
+ int cLevelLast = MINCLEVEL - 1; /* lower than minimum */
unsigned recursive = 0;
unsigned memLimit = 0;
- const char** filenameTable = (const char**)malloc((size_t)argCount * sizeof(const char*)); /* argCount >= 1 */
- unsigned filenameIdx = 0;
+ FileNamesTable* filenames = UTIL_allocateFileNamesTable((size_t)argCount); /* argCount >= 1 */
+ FileNamesTable* file_of_names = UTIL_allocateFileNamesTable((size_t)argCount); /* argCount >= 1 */
const char* programName = argv[0];
const char* outFileName = NULL;
const char* outDirName = NULL;
const char* dictFileName = NULL;
+ const char* patchFromDictFileName = NULL;
const char* suffix = ZSTD_EXTENSION;
unsigned maxDictSize = g_defaultMaxDictSize;
unsigned dictID = 0;
@@ -599,11 +677,6 @@ int main(int argCount, const char* argv[])
size_t srcSizeHint = 0;
int dictCLevel = g_defaultDictCLevel;
unsigned dictSelect = g_defaultSelectivityLevel;
-#ifdef UTIL_HAS_CREATEFILELIST
- const char** extendedFileList = NULL;
- char* fileNamesBuf = NULL;
- unsigned fileNamesNb;
-#endif
#ifndef ZSTD_NODICT
ZDICT_cover_params_t coverParams = defaultCoverParams();
ZDICT_fastCover_params_t fastCoverParams = defaultFastCoverParams();
@@ -617,11 +690,9 @@ int main(int argCount, const char* argv[])
/* init */
(void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */
- (void)memLimit; /* not used when ZSTD_NODECOMPRESS set */
- if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); }
- filenameTable[0] = stdinmark;
- g_displayOut = stderr;
- cLevel = init_cLevel();
+ (void)memLimit;
+ assert(argCount >= 1);
+ if ((filenames==NULL) || (file_of_names==NULL)) { DISPLAY("zstd: allocation error \n"); exit(1); }
programName = lastNameFromPath(programName);
#ifdef ZSTD_MULTITHREAD
nbWorkers = 1;
@@ -649,312 +720,324 @@ int main(int argCount, const char* argv[])
/* command switches */
for (argNb=1; argNb<argCount; argNb++) {
const char* argument = argv[argNb];
- if(!argument) continue; /* Protection if argument empty */
-
- if (nextArgumentsAreFiles==0) {
- /* "-" means stdin/stdout */
- if (!strcmp(argument, "-")){
- if (!filenameIdx) {
- filenameIdx=1, filenameTable[0]=stdinmark;
- outFileName=stdoutmark;
- g_displayLevel-=(g_displayLevel==2);
- continue;
- } }
+ if (!argument) continue; /* Protection if argument empty */
- /* Decode commands (note : aggregated commands are allowed) */
- if (argument[0]=='-') {
-
- if (argument[1]=='-') {
- /* long commands (--long-word) */
- if (!strcmp(argument, "--")) { nextArgumentsAreFiles=1; continue; } /* only file names allowed from now on */
- if (!strcmp(argument, "--list")) { operation=zom_list; continue; }
- if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; }
- if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; }
- if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; }
- if (!strcmp(argument, "--force")) { FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; continue; }
- if (!strcmp(argument, "--version")) { g_displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); }
- if (!strcmp(argument, "--help")) { g_displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); }
- if (!strcmp(argument, "--verbose")) { g_displayLevel++; continue; }
- if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; }
- if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; g_displayLevel-=(g_displayLevel==2); continue; }
- if (!strcmp(argument, "--ultra")) { ultra=1; continue; }
- if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(prefs, 2); continue; }
- if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(prefs, 0); continue; }
- if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(prefs, 2); continue; }
- if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(prefs, 0); continue; }
- if (!strcmp(argument, "--test")) { operation=zom_test; continue; }
- if (!strcmp(argument, "--train")) { operation=zom_train; if (outFileName==NULL) outFileName=g_defaultDictName; continue; }
- if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; lastCommand=1; continue; } /* kept available for compatibility with old syntax ; will be removed one day */
- if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; lastCommand=1; continue; } /* kept available for compatibility with old syntax ; will be removed one day */
- if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(prefs, 0); continue; }
- if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(prefs, 0); continue; }
- if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(prefs, 1); continue; }
- if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
- if (!strcmp(argument, "--output-dir-flat")) {nextArgumentIsOutDirName=1; lastCommand=1; continue; }
- if (!strcmp(argument, "--adapt")) { adapt = 1; continue; }
- if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) CLEAN_RETURN(badusage(programName)); continue; }
- if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; }
- if (!strcmp(argument, "--format=zstd")) { suffix = ZSTD_EXTENSION; FIO_setCompressionType(prefs, FIO_zstdCompression); continue; }
+ if (nextArgumentsAreFiles) {
+ UTIL_refFilename(filenames, argument);
+ continue;
+ }
+
+ /* "-" means stdin/stdout */
+ if (!strcmp(argument, "-")){
+ UTIL_refFilename(filenames, stdinmark);
+ continue;
+ }
+
+ /* Decode commands (note : aggregated commands are allowed) */
+ if (argument[0]=='-') {
+
+ if (argument[1]=='-') {
+ /* long commands (--long-word) */
+ if (!strcmp(argument, "--")) { nextArgumentsAreFiles=1; continue; } /* only file names allowed from now on */
+ if (!strcmp(argument, "--list")) { operation=zom_list; continue; }
+ if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; }
+ if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; }
+ if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; }
+ if (!strcmp(argument, "--force")) { FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; continue; }
+ if (!strcmp(argument, "--version")) { printVersion(); CLEAN_RETURN(0); }
+ if (!strcmp(argument, "--help")) { usage_advanced(programName); CLEAN_RETURN(0); }
+ if (!strcmp(argument, "--verbose")) { g_displayLevel++; continue; }
+ if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; }
+ if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; g_displayLevel-=(g_displayLevel==2); continue; }
+ if (!strcmp(argument, "--ultra")) { ultra=1; continue; }
+ if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(prefs, 2); continue; }
+ if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(prefs, 0); continue; }
+ if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(prefs, 2); continue; }
+ if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(prefs, 0); continue; }
+ if (!strcmp(argument, "--test")) { operation=zom_test; continue; }
+ if (!strcmp(argument, "--train")) { operation=zom_train; if (outFileName==NULL) outFileName=g_defaultDictName; continue; }
+ if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; lastCommand=1; continue; } /* kept available for compatibility with old syntax ; will be removed one day */
+ if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; lastCommand=1; continue; } /* kept available for compatibility with old syntax ; will be removed one day */
+ if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(prefs, 0); continue; }
+ if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(prefs, 0); continue; }
+ if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(prefs, 1); continue; }
+ if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
+ if (!strcmp(argument, "--output-dir-flat")) {nextArgumentIsOutDirName=1; lastCommand=1; continue; }
+ if (!strcmp(argument, "--show-default-cparams")) { showDefaultCParams = 1; continue; }
+ if (!strcmp(argument, "--content-size")) { contentSize = 1; continue; }
+ if (!strcmp(argument, "--no-content-size")) { contentSize = 0; continue; }
+ if (!strcmp(argument, "--adapt")) { adapt = 1; continue; }
+ if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) { badusage(programName); CLEAN_RETURN(1); } continue; }
+ if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; }
+ if (!strcmp(argument, "--format=zstd")) { suffix = ZSTD_EXTENSION; FIO_setCompressionType(prefs, FIO_zstdCompression); continue; }
#ifdef ZSTD_GZCOMPRESS
- if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(prefs, FIO_gzipCompression); continue; }
+ if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(prefs, FIO_gzipCompression); continue; }
#endif
#ifdef ZSTD_LZMACOMPRESS
- if (!strcmp(argument, "--format=lzma")) { suffix = LZMA_EXTENSION; FIO_setCompressionType(prefs, FIO_lzmaCompression); continue; }
- if (!strcmp(argument, "--format=xz")) { suffix = XZ_EXTENSION; FIO_setCompressionType(prefs, FIO_xzCompression); continue; }
+ if (!strcmp(argument, "--format=lzma")) { suffix = LZMA_EXTENSION; FIO_setCompressionType(prefs, FIO_lzmaCompression); continue; }
+ if (!strcmp(argument, "--format=xz")) { suffix = XZ_EXTENSION; FIO_setCompressionType(prefs, FIO_xzCompression); continue; }
#endif
#ifdef ZSTD_LZ4COMPRESS
- if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; FIO_setCompressionType(prefs, FIO_lz4Compression); continue; }
+ if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; FIO_setCompressionType(prefs, FIO_lz4Compression); continue; }
#endif
- if (!strcmp(argument, "--rsyncable")) { rsyncable = 1; continue; }
- if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_lcm_huffman; continue; }
- if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_lcm_uncompressed; continue; }
- if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; }
- if (!strcmp(argument, "--exclude-compressed")) { FIO_setExcludeCompressedFile(prefs, 1); continue; }
- /* long commands with arguments */
+ if (!strcmp(argument, "--rsyncable")) { rsyncable = 1; continue; }
+ if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_lcm_huffman; continue; }
+ if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_lcm_uncompressed; continue; }
+ if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; }
+ if (!strcmp(argument, "--exclude-compressed")) { FIO_setExcludeCompressedFile(prefs, 1); continue; }
+ /* long commands with arguments */
#ifndef ZSTD_NODICT
- if (longCommandWArg(&argument, "--train-cover")) {
- operation = zom_train;
- if (outFileName == NULL)
- outFileName = g_defaultDictName;
- dict = cover;
- /* Allow optional arguments following an = */
- if (*argument == 0) { memset(&coverParams, 0, sizeof(coverParams)); }
- else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); }
- else if (!parseCoverParameters(argument, &coverParams)) { CLEAN_RETURN(badusage(programName)); }
- continue;
- }
- if (longCommandWArg(&argument, "--train-fastcover")) {
- operation = zom_train;
- if (outFileName == NULL)
- outFileName = g_defaultDictName;
- dict = fastCover;
- /* Allow optional arguments following an = */
- if (*argument == 0) { memset(&fastCoverParams, 0, sizeof(fastCoverParams)); }
- else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); }
- else if (!parseFastCoverParameters(argument, &fastCoverParams)) { CLEAN_RETURN(badusage(programName)); }
- continue;
- }
- if (longCommandWArg(&argument, "--train-legacy")) {
- operation = zom_train;
- if (outFileName == NULL)
- outFileName = g_defaultDictName;
- dict = legacy;
- /* Allow optional arguments following an = */
- if (*argument == 0) { continue; }
- else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); }
- else if (!parseLegacyParameters(argument, &dictSelect)) { CLEAN_RETURN(badusage(programName)); }
- continue;
- }
+ if (longCommandWArg(&argument, "--train-cover")) {
+ operation = zom_train;
+ if (outFileName == NULL)
+ outFileName = g_defaultDictName;
+ dict = cover;
+ /* Allow optional arguments following an = */
+ if (*argument == 0) { memset(&coverParams, 0, sizeof(coverParams)); }
+ else if (*argument++ != '=') { badusage(programName); CLEAN_RETURN(1); }
+ else if (!parseCoverParameters(argument, &coverParams)) { badusage(programName); CLEAN_RETURN(1); }
+ continue;
+ }
+ if (longCommandWArg(&argument, "--train-fastcover")) {
+ operation = zom_train;
+ if (outFileName == NULL)
+ outFileName = g_defaultDictName;
+ dict = fastCover;
+ /* Allow optional arguments following an = */
+ if (*argument == 0) { memset(&fastCoverParams, 0, sizeof(fastCoverParams)); }
+ else if (*argument++ != '=') { badusage(programName); CLEAN_RETURN(1); }
+ else if (!parseFastCoverParameters(argument, &fastCoverParams)) { badusage(programName); CLEAN_RETURN(1); }
+ continue;
+ }
+ if (longCommandWArg(&argument, "--train-legacy")) {
+ operation = zom_train;
+ if (outFileName == NULL)
+ outFileName = g_defaultDictName;
+ dict = legacy;
+ /* Allow optional arguments following an = */
+ if (*argument == 0) { continue; }
+ else if (*argument++ != '=') { badusage(programName); CLEAN_RETURN(1); }
+ else if (!parseLegacyParameters(argument, &dictSelect)) { badusage(programName); CLEAN_RETURN(1); }
+ continue;
+ }
#endif
- if (longCommandWArg(&argument, "--threads=")) { nbWorkers = (int)readU32FromChar(&argument); continue; }
- if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; }
- if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; }
- if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; }
- if (longCommandWArg(&argument, "--block-size=")) { blockSize = readU32FromChar(&argument); continue; }
- if (longCommandWArg(&argument, "--maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; }
- if (longCommandWArg(&argument, "--dictID=")) { dictID = readU32FromChar(&argument); continue; }
- if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) CLEAN_RETURN(badusage(programName)); continue; }
- if (longCommandWArg(&argument, "--stream-size=")) { streamSrcSize = readU32FromChar(&argument); continue; }
- if (longCommandWArg(&argument, "--target-compressed-block-size=")) { targetCBlockSize = readU32FromChar(&argument); continue; }
- if (longCommandWArg(&argument, "--size-hint=")) { srcSizeHint = readU32FromChar(&argument); continue; }
- if (longCommandWArg(&argument, "--output-dir-flat=")) { outDirName = argument; continue; }
- if (longCommandWArg(&argument, "--long")) {
- unsigned ldmWindowLog = 0;
- ldmFlag = 1;
- /* Parse optional window log */
- if (*argument == '=') {
- ++argument;
- ldmWindowLog = readU32FromChar(&argument);
- } else if (*argument != 0) {
- /* Invalid character following --long */
- CLEAN_RETURN(badusage(programName));
- }
- /* Only set windowLog if not already set by --zstd */
- if (compressionParams.windowLog == 0)
- compressionParams.windowLog = ldmWindowLog;
- continue;
+ if (longCommandWArg(&argument, "--threads=")) { nbWorkers = (int)readU32FromChar(&argument); continue; }
+ if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; }
+ if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; }
+ if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; }
+ if (longCommandWArg(&argument, "--block-size=")) { blockSize = readSizeTFromChar(&argument); continue; }
+ if (longCommandWArg(&argument, "--maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; }
+ if (longCommandWArg(&argument, "--dictID=")) { dictID = readU32FromChar(&argument); continue; }
+ if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) { badusage(programName); CLEAN_RETURN(1); } continue; }
+ if (longCommandWArg(&argument, "--stream-size=")) { streamSrcSize = readSizeTFromChar(&argument); continue; }
+ if (longCommandWArg(&argument, "--target-compressed-block-size=")) { targetCBlockSize = readSizeTFromChar(&argument); continue; }
+ if (longCommandWArg(&argument, "--size-hint=")) { srcSizeHint = readSizeTFromChar(&argument); continue; }
+ if (longCommandWArg(&argument, "--output-dir-flat=")) { outDirName = argument; continue; }
+ if (longCommandWArg(&argument, "--patch-from=")) { patchFromDictFileName = argument; continue; }
+ if (longCommandWArg(&argument, "--long")) {
+ unsigned ldmWindowLog = 0;
+ ldmFlag = 1;
+ /* Parse optional window log */
+ if (*argument == '=') {
+ ++argument;
+ ldmWindowLog = readU32FromChar(&argument);
+ } else if (*argument != 0) {
+ /* Invalid character following --long */
+ badusage(programName);
+ CLEAN_RETURN(1);
}
+ /* Only set windowLog if not already set by --zstd */
+ if (compressionParams.windowLog == 0)
+ compressionParams.windowLog = ldmWindowLog;
+ continue;
+ }
#ifndef ZSTD_NOCOMPRESS /* linking ZSTD_minCLevel() requires compression support */
- if (longCommandWArg(&argument, "--fast")) {
- /* Parse optional acceleration factor */
- if (*argument == '=') {
- U32 const maxFast = (U32)-ZSTD_minCLevel();
- U32 fastLevel;
- ++argument;
- fastLevel = readU32FromChar(&argument);
- if (fastLevel > maxFast) fastLevel = maxFast;
- if (fastLevel) {
- dictCLevel = cLevel = -(int)fastLevel;
- } else {
- CLEAN_RETURN(badusage(programName));
- }
- } else if (*argument != 0) {
- /* Invalid character following --fast */
- CLEAN_RETURN(badusage(programName));
+ if (longCommandWArg(&argument, "--fast")) {
+ /* Parse optional acceleration factor */
+ if (*argument == '=') {
+ U32 const maxFast = (U32)-ZSTD_minCLevel();
+ U32 fastLevel;
+ ++argument;
+ fastLevel = readU32FromChar(&argument);
+ if (fastLevel > maxFast) fastLevel = maxFast;
+ if (fastLevel) {
+ dictCLevel = cLevel = -(int)fastLevel;
} else {
- cLevel = -1; /* default for --fast */
+ badusage(programName);
+ CLEAN_RETURN(1);
}
- continue;
+ } else if (*argument != 0) {
+ /* Invalid character following --fast */
+ badusage(programName);
+ CLEAN_RETURN(1);
+ } else {
+ cLevel = -1; /* default for --fast */
}
+ continue;
+ }
#endif
- /* fall-through, will trigger bad_usage() later on */
+
+ if (longCommandWArg(&argument, "--filelist=")) {
+ UTIL_refFilename(file_of_names, argument);
+ continue;
}
- argument++;
- while (argument[0]!=0) {
- if (lastCommand) {
- DISPLAY("error : command must be followed by argument \n");
- CLEAN_RETURN(1);
- }
+ /* fall-through, will trigger bad_usage() later on */
+ }
+
+ argument++;
+ while (argument[0]!=0) {
+ if (lastCommand) {
+ DISPLAY("error : command must be followed by argument \n");
+ CLEAN_RETURN(1);
+ }
#ifndef ZSTD_NOCOMPRESS
- /* compression Level */
- if ((*argument>='0') && (*argument<='9')) {
- dictCLevel = cLevel = (int)readU32FromChar(&argument);
- continue;
- }
+ /* compression Level */
+ if ((*argument>='0') && (*argument<='9')) {
+ dictCLevel = cLevel = (int)readU32FromChar(&argument);
+ continue;
+ }
#endif
- switch(argument[0])
- {
- /* Display help */
- case 'V': g_displayOut=stdout; printVersion(); CLEAN_RETURN(0); /* Version Only */
- case 'H':
- case 'h': g_displayOut=stdout; CLEAN_RETURN(usage_advanced(programName));
+ switch(argument[0])
+ {
+ /* Display help */
+ case 'V': printVersion(); CLEAN_RETURN(0); /* Version Only */
+ case 'H':
+ case 'h': usage_advanced(programName); CLEAN_RETURN(0);
- /* Compress */
- case 'z': operation=zom_compress; argument++; break;
+ /* Compress */
+ case 'z': operation=zom_compress; argument++; break;
- /* Decoding */
- case 'd':
+ /* Decoding */
+ case 'd':
#ifndef ZSTD_NOBENCH
- benchParams.mode = BMK_decodeOnly;
- if (operation==zom_bench) { argument++; break; } /* benchmark decode (hidden option) */
+ benchParams.mode = BMK_decodeOnly;
+ if (operation==zom_bench) { argument++; break; } /* benchmark decode (hidden option) */
#endif
- operation=zom_decompress; argument++; break;
+ operation=zom_decompress; argument++; break;
- /* Force stdout, even if stdout==console */
- case 'c': forceStdout=1; outFileName=stdoutmark; argument++; break;
+ /* Force stdout, even if stdout==console */
+ case 'c': forceStdout=1; outFileName=stdoutmark; argument++; break;
- /* Use file content as dictionary */
- case 'D': nextEntryIsDictionary = 1; lastCommand = 1; argument++; break;
+ /* Use file content as dictionary */
+ case 'D': nextEntryIsDictionary = 1; lastCommand = 1; argument++; break;
- /* Overwrite */
- case 'f': FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; argument++; break;
+ /* Overwrite */
+ case 'f': FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; argument++; break;
- /* Verbose mode */
- case 'v': g_displayLevel++; argument++; break;
+ /* Verbose mode */
+ case 'v': g_displayLevel++; argument++; break;
- /* Quiet mode */
- case 'q': g_displayLevel--; argument++; break;
+ /* Quiet mode */
+ case 'q': g_displayLevel--; argument++; break;
- /* keep source file (default) */
- case 'k': FIO_setRemoveSrcFile(prefs, 0); argument++; break;
+ /* keep source file (default) */
+ case 'k': FIO_setRemoveSrcFile(prefs, 0); argument++; break;
- /* Checksum */
- case 'C': FIO_setChecksumFlag(prefs, 2); argument++; break;
+ /* Checksum */
+ case 'C': FIO_setChecksumFlag(prefs, 2); argument++; break;
- /* test compressed file */
- case 't': operation=zom_test; argument++; break;
+ /* test compressed file */
+ case 't': operation=zom_test; argument++; break;
- /* destination file name */
- case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break;
+ /* destination file name */
+ case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break;
- /* limit decompression memory */
- case 'M':
- argument++;
- memLimit = readU32FromChar(&argument);
- break;
- case 'l': operation=zom_list; argument++; break;
+ /* limit memory */
+ case 'M':
+ argument++;
+ memLimit = readU32FromChar(&argument);
+ break;
+ case 'l': operation=zom_list; argument++; break;
#ifdef UTIL_HAS_CREATEFILELIST
- /* recursive */
- case 'r': recursive=1; argument++; break;
+ /* recursive */
+ case 'r': recursive=1; argument++; break;
#endif
#ifndef ZSTD_NOBENCH
- /* Benchmark */
- case 'b':
- operation=zom_bench;
- argument++;
- break;
-
- /* range bench (benchmark only) */
- case 'e':
- /* compression Level */
- argument++;
- cLevelLast = (int)readU32FromChar(&argument);
- break;
-
- /* Modify Nb Iterations (benchmark only) */
- case 'i':
- argument++;
- bench_nbSeconds = readU32FromChar(&argument);
- break;
-
- /* cut input into blocks (benchmark only) */
- case 'B':
- argument++;
- blockSize = readU32FromChar(&argument);
- break;
-
- /* benchmark files separately (hidden option) */
- case 'S':
- argument++;
- separateFiles = 1;
- break;
+ /* Benchmark */
+ case 'b':
+ operation=zom_bench;
+ argument++;
+ break;
+
+ /* range bench (benchmark only) */
+ case 'e':
+ /* compression Level */
+ argument++;
+ cLevelLast = (int)readU32FromChar(&argument);
+ break;
+
+ /* Modify Nb Iterations (benchmark only) */
+ case 'i':
+ argument++;
+ bench_nbSeconds = readU32FromChar(&argument);
+ break;
+
+ /* cut input into blocks (benchmark only) */
+ case 'B':
+ argument++;
+ blockSize = readU32FromChar(&argument);
+ break;
+
+ /* benchmark files separately (hidden option) */
+ case 'S':
+ argument++;
+ separateFiles = 1;
+ break;
#endif /* ZSTD_NOBENCH */
- /* nb of threads (hidden option) */
- case 'T':
- argument++;
- nbWorkers = (int)readU32FromChar(&argument);
- break;
+ /* nb of threads (hidden option) */
+ case 'T':
+ argument++;
+ nbWorkers = (int)readU32FromChar(&argument);
+ break;
- /* Dictionary Selection level */
- case 's':
- argument++;
- dictSelect = readU32FromChar(&argument);
- break;
+ /* Dictionary Selection level */
+ case 's':
+ argument++;
+ dictSelect = readU32FromChar(&argument);
+ break;
- /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */
- case 'p': argument++;
+ /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */
+ case 'p': argument++;
#ifndef ZSTD_NOBENCH
- if ((*argument>='0') && (*argument<='9')) {
- benchParams.additionalParam = (int)readU32FromChar(&argument);
- } else
+ if ((*argument>='0') && (*argument<='9')) {
+ benchParams.additionalParam = (int)readU32FromChar(&argument);
+ } else
#endif
- main_pause=1;
- break;
-
- /* Select compressibility of synthetic sample */
- case 'P':
- { argument++;
- compressibility = (double)readU32FromChar(&argument) / 100;
- }
+ main_pause=1;
break;
- /* unknown command */
- default : CLEAN_RETURN(badusage(programName));
- }
+ /* Select compressibility of synthetic sample */
+ case 'P':
+ { argument++;
+ compressibility = (double)readU32FromChar(&argument) / 100;
}
- continue;
- } /* if (argument[0]=='-') */
-
- if (nextArgumentIsMaxDict) { /* kept available for compatibility with old syntax ; will be removed one day */
- nextArgumentIsMaxDict = 0;
- lastCommand = 0;
- maxDictSize = readU32FromChar(&argument);
- continue;
- }
+ break;
- if (nextArgumentIsDictID) { /* kept available for compatibility with old syntax ; will be removed one day */
- nextArgumentIsDictID = 0;
- lastCommand = 0;
- dictID = readU32FromChar(&argument);
- continue;
+ /* unknown command */
+ default : badusage(programName); CLEAN_RETURN(1);
+ }
}
+ continue;
+ } /* if (argument[0]=='-') */
+
+ if (nextArgumentIsMaxDict) { /* kept available for compatibility with old syntax ; will be removed one day */
+ nextArgumentIsMaxDict = 0;
+ lastCommand = 0;
+ maxDictSize = readU32FromChar(&argument);
+ continue;
+ }
- } /* if (nextArgumentIsAFile==0) */
+ if (nextArgumentIsDictID) { /* kept available for compatibility with old syntax ; will be removed one day */
+ nextArgumentIsDictID = 0;
+ lastCommand = 0;
+ dictID = readU32FromChar(&argument);
+ continue;
+ }
if (nextEntryIsDictionary) {
nextEntryIsDictionary = 0;
@@ -978,8 +1061,8 @@ int main(int argCount, const char* argv[])
continue;
}
- /* add filename to list */
- filenameTable[filenameIdx++] = argument;
+ /* none of the above : add filename to list */
+ UTIL_refFilename(filenames, argument);
}
if (lastCommand) { /* forgotten argument */
@@ -1003,31 +1086,37 @@ int main(int argCount, const char* argv[])
#ifdef UTIL_HAS_CREATEFILELIST
g_utilDisplayLevel = g_displayLevel;
if (!followLinks) {
- unsigned u;
- for (u=0, fileNamesNb=0; u<filenameIdx; u++) {
- if (UTIL_isLink(filenameTable[u])
-#ifndef _MSC_VER
- && !UTIL_isFIFO(filenameTable[u])
-#endif /* _MSC_VER */
+ unsigned u, fileNamesNb;
+ unsigned const nbFilenames = (unsigned)filenames->tableSize;
+ for (u=0, fileNamesNb=0; u<nbFilenames; u++) {
+ if ( UTIL_isLink(filenames->fileNames[u])
+ && !UTIL_isFIFO(filenames->fileNames[u])
) {
- DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", filenameTable[u]);
+ DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring \n", filenames->fileNames[u]);
} else {
- filenameTable[fileNamesNb++] = filenameTable[u];
+ filenames->fileNames[fileNamesNb++] = filenames->fileNames[u];
+ } }
+ if (fileNamesNb == 0 && nbFilenames > 0) /* all names are eliminated */
+ CLEAN_RETURN(1);
+ filenames->tableSize = fileNamesNb;
+ } /* if (!followLinks) */
+
+ /* read names from a file */
+ if (file_of_names->tableSize) {
+ size_t const nbFileLists = file_of_names->tableSize;
+ size_t flNb;
+ for (flNb=0; flNb < nbFileLists; flNb++) {
+ FileNamesTable* const fnt = UTIL_createFileNamesTable_fromFileName(file_of_names->fileNames[flNb]);
+ if (fnt==NULL) {
+ DISPLAYLEVEL(1, "zstd: error reading %s \n", file_of_names->fileNames[flNb]);
+ CLEAN_RETURN(1);
}
+ filenames = UTIL_mergeFileNamesTable(filenames, fnt);
}
- if (fileNamesNb == 0 && filenameIdx > 0)
- CLEAN_RETURN(1);
- filenameIdx = fileNamesNb;
}
+
if (recursive) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */
- extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb, followLinks);
- if (extendedFileList) {
- unsigned u;
- for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, extendedFileList[u]);
- free((void*)filenameTable);
- filenameTable = extendedFileList;
- filenameIdx = fileNamesNb;
- }
+ UTIL_expandFNT(&filenames, followLinks);
}
#else
(void)followLinks;
@@ -1035,7 +1124,7 @@ int main(int argCount, const char* argv[])
if (operation == zom_list) {
#ifndef ZSTD_NODECOMPRESS
- int const ret = FIO_listMultipleFiles(filenameIdx, filenameTable, g_displayLevel);
+ int const ret = FIO_listMultipleFiles((unsigned)filenames->tableSize, filenames->fileNames, g_displayLevel);
CLEAN_RETURN(ret);
#else
DISPLAY("file information is not supported \n");
@@ -1066,26 +1155,23 @@ int main(int argCount, const char* argv[])
if (cLevelLast < cLevel) cLevelLast = cLevel;
if (cLevelLast > cLevel)
DISPLAYLEVEL(3, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
- if(filenameIdx) {
+ if (filenames->tableSize > 0) {
if(separateFiles) {
unsigned i;
- for(i = 0; i < filenameIdx; i++) {
+ for(i = 0; i < filenames->tableSize; i++) {
int c;
- DISPLAYLEVEL(3, "Benchmarking %s \n", filenameTable[i]);
+ DISPLAYLEVEL(3, "Benchmarking %s \n", filenames->fileNames[i]);
for(c = cLevel; c <= cLevelLast; c++) {
- BMK_benchFilesAdvanced(&filenameTable[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams);
- }
- }
+ BMK_benchFilesAdvanced(&filenames->fileNames[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams);
+ } }
} else {
for(; cLevel <= cLevelLast; cLevel++) {
- BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams);
- }
- }
+ BMK_benchFilesAdvanced(filenames->fileNames, (unsigned)filenames->tableSize, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams);
+ } }
} else {
for(; cLevel <= cLevelLast; cLevel++) {
BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &benchParams);
- }
- }
+ } }
#else
(void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; (void)compressibility;
@@ -1104,18 +1190,18 @@ int main(int argCount, const char* argv[])
int const optimize = !coverParams.k || !coverParams.d;
coverParams.nbThreads = (unsigned)nbWorkers;
coverParams.zParams = zParams;
- operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, NULL, optimize);
+ operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (unsigned)filenames->tableSize, blockSize, NULL, &coverParams, NULL, optimize);
} else if (dict == fastCover) {
int const optimize = !fastCoverParams.k || !fastCoverParams.d;
fastCoverParams.nbThreads = (unsigned)nbWorkers;
fastCoverParams.zParams = zParams;
- operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, NULL, &fastCoverParams, optimize);
+ operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (unsigned)filenames->tableSize, blockSize, NULL, NULL, &fastCoverParams, optimize);
} else {
ZDICT_legacy_params_t dictParams;
memset(&dictParams, 0, sizeof(dictParams));
dictParams.selectivityLevel = dictSelect;
dictParams.zParams = zParams;
- operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, &dictParams, NULL, NULL, 0);
+ operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (unsigned)filenames->tableSize, blockSize, &dictParams, NULL, NULL, 0);
}
#else
(void)dictCLevel; (void)dictSelect; (void)dictID; (void)maxDictSize; /* not used when ZSTD_NODICT set */
@@ -1130,19 +1216,23 @@ int main(int argCount, const char* argv[])
#endif
/* No input filename ==> use stdin and stdout */
- filenameIdx += !filenameIdx; /* filenameTable[0] is stdin by default */
- if (!strcmp(filenameTable[0], stdinmark) && !outFileName)
+ if (filenames->tableSize == 0) UTIL_refFilename(filenames, stdinmark);
+ if (!strcmp(filenames->fileNames[0], stdinmark) && !outFileName)
outFileName = stdoutmark; /* when input is stdin, default output is stdout */
/* Check if input/output defined as console; trigger an error in this case */
- if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) )
- CLEAN_RETURN(badusage(programName));
+ if (!strcmp(filenames->fileNames[0], stdinmark) && IS_CONSOLE(stdin) ) {
+ badusage(programName);
+ CLEAN_RETURN(1);
+ }
if ( outFileName && !strcmp(outFileName, stdoutmark)
&& IS_CONSOLE(stdout)
- && !strcmp(filenameTable[0], stdinmark)
+ && !strcmp(filenames->fileNames[0], stdinmark)
&& !forceStdout
- && operation!=zom_decompress )
- CLEAN_RETURN(badusage(programName));
+ && operation!=zom_decompress ) {
+ badusage(programName);
+ CLEAN_RETURN(1);
+ }
#ifndef ZSTD_NOCOMPRESS
/* check compression level limits */
@@ -1153,14 +1243,42 @@ int main(int argCount, const char* argv[])
} }
#endif
+ if (showDefaultCParams) {
+ if (operation == zom_decompress) {
+ DISPLAY("error : can't use --show-default-cparams in decomrpession mode \n");
+ CLEAN_RETURN(1);
+ }
+ }
+
+ if (dictFileName != NULL && patchFromDictFileName != NULL) {
+ DISPLAY("error : can't use -D and --patch-from=# at the same time \n");
+ CLEAN_RETURN(1);
+ }
+
+ if (patchFromDictFileName != NULL && filenames->tableSize > 1) {
+ DISPLAY("error : can't use --patch-from=# on multiple files \n");
+ CLEAN_RETURN(1);
+ }
+
/* No status message in pipe mode (stdin - stdout) or multi-files mode */
- if (!strcmp(filenameTable[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (g_displayLevel==2)) g_displayLevel=1;
- if ((filenameIdx>1) & (g_displayLevel==2)) g_displayLevel=1;
+ if (!strcmp(filenames->fileNames[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (g_displayLevel==2)) g_displayLevel=1;
+ if ((filenames->tableSize > 1) & (g_displayLevel==2)) g_displayLevel=1;
/* IO Stream/File */
FIO_setNotificationLevel(g_displayLevel);
+ FIO_setPatchFromMode(prefs, patchFromDictFileName != NULL);
+ if (memLimit == 0) {
+ if (compressionParams.windowLog == 0) {
+ memLimit = (U32)1 << g_defaultMaxWindowLog;
+ } else {
+ memLimit = (U32)1 << (compressionParams.windowLog & 31);
+ } }
+ if (patchFromDictFileName != NULL)
+ dictFileName = patchFromDictFileName;
+ FIO_setMemLimit(prefs, memLimit);
if (operation==zom_compress) {
#ifndef ZSTD_NOCOMPRESS
+ FIO_setContentSize(prefs, contentSize);
FIO_setNbWorkers(prefs, nbWorkers);
FIO_setBlockSize(prefs, (int)blockSize);
if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, (int)g_overlapLog);
@@ -1180,28 +1298,45 @@ int main(int argCount, const char* argv[])
if (adaptMin > cLevel) cLevel = adaptMin;
if (adaptMax < cLevel) cLevel = adaptMax;
- if ((filenameIdx==1) && outFileName)
- operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams);
+ /* Compare strategies constant with the ground truth */
+ { ZSTD_bounds strategyBounds = ZSTD_cParam_getBounds(ZSTD_c_strategy);
+ assert(ZSTD_NB_STRATEGIES == strategyBounds.upperBound);
+ (void)strategyBounds; }
+
+ if (showDefaultCParams) {
+ size_t fileNb;
+ for (fileNb = 0; fileNb < (size_t)filenames->tableSize; fileNb++) {
+ unsigned long long fileSize = UTIL_getFileSize(filenames->fileNames[fileNb]);
+ const size_t dictSize = dictFileName != NULL ? (size_t)UTIL_getFileSize(dictFileName) : 0;
+ const ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, fileSize, dictSize);
+ if (fileSize != UTIL_FILESIZE_UNKNOWN) DISPLAY("%s (%u bytes)\n", filenames->fileNames[fileNb], (unsigned)fileSize);
+ else DISPLAY("%s (src size unknown)\n", filenames->fileNames[fileNb]);
+ DISPLAY(" - windowLog : %u\n", cParams.windowLog);
+ DISPLAY(" - chainLog : %u\n", cParams.chainLog);
+ DISPLAY(" - hashLog : %u\n", cParams.hashLog);
+ DISPLAY(" - searchLog : %u\n", cParams.searchLog);
+ DISPLAY(" - minMatch : %u\n", cParams.minMatch);
+ DISPLAY(" - targetLength : %u\n", cParams.targetLength);
+ assert(cParams.strategy < ZSTD_NB_STRATEGIES + 1);
+ DISPLAY(" - strategy : %s (%u)\n", ZSTD_strategyMap[(int)cParams.strategy], (unsigned)cParams.strategy);
+ }
+ }
+
+ if ((filenames->tableSize==1) && outFileName)
+ operationResult = FIO_compressFilename(prefs, outFileName, filenames->fileNames[0], dictFileName, cLevel, compressionParams);
else
- operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams);
+ operationResult = FIO_compressMultipleFilenames(prefs, filenames->fileNames, (unsigned)filenames->tableSize, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams);
#else
- (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; /* not used when ZSTD_NOCOMPRESS set */
+ (void)contentSize; (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; (void)ZSTD_strategyMap; /* not used when ZSTD_NOCOMPRESS set */
DISPLAY("Compression not supported \n");
#endif
} else { /* decompression or test */
#ifndef ZSTD_NODECOMPRESS
- if (memLimit == 0) {
- if (compressionParams.windowLog == 0)
- memLimit = (U32)1 << g_defaultMaxWindowLog;
- else {
- memLimit = (U32)1 << (compressionParams.windowLog & 31);
- }
+ if (filenames->tableSize == 1 && outFileName) {
+ operationResult = FIO_decompressFilename(prefs, outFileName, filenames->fileNames[0], dictFileName);
+ } else {
+ operationResult = FIO_decompressMultipleFilenames(prefs, filenames->fileNames, (unsigned)filenames->tableSize, outDirName, outFileName, dictFileName);
}
- FIO_setMemLimit(prefs, memLimit);
- if (filenameIdx==1 && outFileName)
- operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName);
- else
- operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, outFileName, dictFileName);
#else
DISPLAY("Decompression not supported \n");
#endif
@@ -1209,13 +1344,9 @@ int main(int argCount, const char* argv[])
_end:
FIO_freePreferences(prefs);
-
if (main_pause) waitEnter();
-#ifdef UTIL_HAS_CREATEFILELIST
- if (extendedFileList)
- UTIL_freeFileList(extendedFileList, fileNamesBuf);
- else
-#endif
- free((void*)filenameTable);
+ UTIL_freeFileNamesTable(filenames);
+ UTIL_freeFileNamesTable(file_of_names);
+
return operationResult;
}
diff --git a/programs/zstdgrep b/programs/zstdgrep
index 4879fb0dae49..61efaa9474a0 100755
--- a/programs/zstdgrep
+++ b/programs/zstdgrep
@@ -109,7 +109,7 @@ if [ "$#" -lt 1 ]; then
# ... on stdin
set -f # Disable file name generation (globbing).
# shellcheck disable=SC2086
- "${zcat}" -fq - | "${grep}" ${grep_args} -- "${pattern}" -
+ "${zcat}" - | "${grep}" ${grep_args} -- "${pattern}" -
EXIT_CODE=$?
set +f
else
@@ -121,9 +121,9 @@ else
while [ "$#" -gt 0 ]; do
# shellcheck disable=SC2086
if [ $pattern_found -eq 2 ]; then
- "${zcat}" -fq -- "$1" | "${grep}" --label="${1}" ${grep_args} -- -
+ "${zcat}" -- "$1" | "${grep}" --label="${1}" ${grep_args} -- -
else
- "${zcat}" -fq -- "$1" | "${grep}" --label="${1}" ${grep_args} -- "${pattern}" -
+ "${zcat}" -- "$1" | "${grep}" --label="${1}" ${grep_args} -- "${pattern}" -
fi
[ "$?" -ne 0 ] && EXIT_CODE=1
shift
diff --git a/programs/zstdgrep.1 b/programs/zstdgrep.1
index b97f8cabcc00..fe1d29bbb062 100644
--- a/programs/zstdgrep.1
+++ b/programs/zstdgrep.1
@@ -1,5 +1,5 @@
.
-.TH "ZSTDGREP" "1" "October 2019" "zstd 1.4.4" "User Commands"
+.TH "ZSTDGREP" "1" "May 2020" "zstd 1.4.5" "User Commands"
.
.SH "NAME"
\fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files
diff --git a/programs/zstdless.1 b/programs/zstdless.1
index 1ecc8bdc531c..d54c6400cee2 100644
--- a/programs/zstdless.1
+++ b/programs/zstdless.1
@@ -1,5 +1,5 @@
.
-.TH "ZSTDLESS" "1" "October 2019" "zstd 1.4.4" "User Commands"
+.TH "ZSTDLESS" "1" "May 2020" "zstd 1.4.5" "User Commands"
.
.SH "NAME"
\fBzstdless\fR \- view zstandard\-compressed files
generated by cgit v1.2.3 (git 2.25.1) at 2025-07-30 18:10:50 +0000