summaryrefslogtreecommitdiff
path: root/programs
diff options
context:
space:
mode:
Diffstat (limited to 'programs')
-rw-r--r--programs/.gitignore32
-rw-r--r--programs/BUCK63
-rw-r--r--programs/Makefile227
-rw-r--r--programs/README.md94
-rw-r--r--programs/bench.c595
-rw-r--r--programs/bench.h29
-rw-r--r--programs/datagen.c180
-rw-r--r--programs/datagen.h27
-rw-r--r--programs/dibio.c306
-rw-r--r--programs/dibio.h38
-rw-r--r--programs/fileio.c1200
-rw-r--r--programs/fileio.h94
-rw-r--r--programs/platform.h137
-rw-r--r--programs/util.h473
-rw-r--r--programs/windres/generate_res.bat11
-rw-r--r--programs/windres/verrsrc.h8
-rw-r--r--programs/windres/zstd.rc51
-rw-r--r--programs/windres/zstd32.resbin0 -> 1044 bytes
-rw-r--r--programs/windres/zstd64.resbin0 -> 1044 bytes
-rw-r--r--programs/zstd.1408
-rw-r--r--programs/zstdcli.c696
-rwxr-xr-xprograms/zstdgrep124
-rwxr-xr-xprograms/zstdless2
23 files changed, 4795 insertions, 0 deletions
diff --git a/programs/.gitignore b/programs/.gitignore
new file mode 100644
index 000000000000..eeaf051d6edf
--- /dev/null
+++ b/programs/.gitignore
@@ -0,0 +1,32 @@
+# local binary (Makefile)
+zstd
+zstd32
+zstd-compress
+zstd-decompress
+
+# Object files
+*.o
+*.ko
+default.profraw
+have_zlib
+
+# Executables
+*.exe
+*.out
+*.app
+
+# Default result files
+dictionary
+grillResults.txt
+_*
+tmp*
+*.zst
+result
+out
+
+# fuzzer
+afl
+
+# Misc files
+*.bat
+dirTest*
diff --git a/programs/BUCK b/programs/BUCK
new file mode 100644
index 000000000000..069403042993
--- /dev/null
+++ b/programs/BUCK
@@ -0,0 +1,63 @@
+cxx_binary(
+ name='zstd',
+ headers=glob(['*.h'], excludes=['datagen.h', 'platform.h', 'util.h']),
+ srcs=glob(['*.c'], excludes=['datagen.c']),
+ deps=[
+ ':datagen',
+ ':util',
+ '//lib:zstd',
+ '//lib:zdict',
+ '//lib:mem',
+ '//lib:xxhash',
+ ],
+)
+
+cxx_binary(
+ name='zstdmt',
+ headers=glob(['*.h'], excludes=['datagen.h', 'platform.h', 'util.h']),
+ srcs=glob(['*.c'], excludes=['datagen.c']),
+ deps=[
+ ':datagen',
+ ':util',
+ '//lib:zstd',
+ '//lib:zdict',
+ '//lib:mem',
+ '//lib:xxhash',
+ ],
+ preprocessor_flags=['-DZSTD_MULTITHREAD'],
+ linker_flags=['-lpthread'],
+)
+
+cxx_binary(
+ name='gzstd',
+ headers=glob(['*.h'], excludes=['datagen.h', 'platform.h', 'util.h']),
+ srcs=glob(['*.c'], excludes=['datagen.c']),
+ deps=[
+ ':datagen',
+ ':util',
+ '//lib:zstd',
+ '//lib:zdict',
+ '//lib:mem',
+ '//lib:xxhash',
+ ],
+ preprocessor_flags=['-DZSTD_GZDECOMPRESS'],
+ linker_flags=['-lz'],
+)
+
+cxx_library(
+ name='datagen',
+ visibility=['PUBLIC'],
+ header_namespace='',
+ exported_headers=['datagen.h'],
+ srcs=['datagen.c'],
+ deps=['//lib:mem'],
+)
+
+
+cxx_library(
+ name='util',
+ visibility=['PUBLIC'],
+ header_namespace='',
+ exported_headers=['util.h', 'platform.h'],
+ deps=['//lib:mem'],
+)
diff --git a/programs/Makefile b/programs/Makefile
new file mode 100644
index 000000000000..1475cb610916
--- /dev/null
+++ b/programs/Makefile
@@ -0,0 +1,227 @@
+# ##########################################################################
+# Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
+# All rights reserved.
+#
+# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+# ##########################################################################
+# zstd : Command Line Utility, supporting gzip-like arguments
+# zstd32 : Same as zstd, but forced to compile in 32-bits mode
+# zstd_nolegacy : zstd without support of decompression of legacy versions
+# zstd-small : minimal zstd without dictionary builder and benchmark
+# zstd-compress : compressor-only version of zstd
+# zstd-decompress : decompressor-only version of zstd
+# ##########################################################################
+
+ZSTDDIR = ../lib
+
+ifeq ($(shell $(CC) -v 2>&1 | grep -c "gcc version "), 1)
+ALIGN_LOOP = -falign-loops=32
+else
+ALIGN_LOOP =
+endif
+
+CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
+ -I$(ZSTDDIR)/dictBuilder \
+ -DXXH_NAMESPACE=ZSTD_ # because xxhash.o already compiled with this macro from library
+CFLAGS ?= -O3
+DEBUGFLAGS = -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
+ -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security
+CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
+FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
+
+
+ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
+ZSTDCOMP_FILES := $(ZSTDDIR)/compress/*.c
+ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c
+ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
+ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
+ZSTDDECOMP_O = $(ZSTDDIR)/decompress/zstd_decompress.o
+
+ZSTD_LEGACY_SUPPORT ?= 4
+ZSTDLEGACY_FILES:=
+ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
+ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
+ ZSTDLEGACY_FILES += $(shell ls $(ZSTDDIR)/legacy/*.c | grep 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
+endif
+ CPPFLAGS += -I$(ZSTDDIR)/legacy
+else
+endif
+
+ZSTDLIB_FILES := $(wildcard $(ZSTD_FILES)) $(wildcard $(ZSTDLEGACY_FILES)) $(wildcard $(ZDICT_FILES))
+ZSTDLIB_OBJ := $(patsubst %.c,%.o,$(ZSTDLIB_FILES))
+
+# Define *.exe as extension for Windows systems
+ifneq (,$(filter Windows%,$(OS)))
+EXT =.exe
+RES64_FILE = windres/zstd64.res
+RES32_FILE = windres/zstd32.res
+ifneq (,$(filter x86_64%,$(shell $(CC) -dumpmachine)))
+ RES_FILE = $(RES64_FILE)
+else
+ RES_FILE = $(RES32_FILE)
+endif
+else
+EXT =
+endif
+
+# zlib detection
+NO_ZLIB_MSG := ==> no zlib, building zstd without .gz support
+VOID = /dev/null
+HAVE_ZLIB := $(shell printf '\#include <zlib.h>\nint main(){}' | $(CC) -o have_zlib -x c - -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0)
+ifeq ($(HAVE_ZLIB), 1)
+ZLIB_MSG := ==> building zstd with .gz compression support
+ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS
+ZLIBLD = -lz
+else
+ZLIB_MSG := $(NO_ZLIB_MSG)
+endif
+# lzma detection
+NO_LZMA_MSG := ==> no liblzma, building zstd without .xz/.lzma support
+HAVE_LZMA := $(shell printf '\#include <lzma.h>\nint main(){}' | $(CC) -o have_lzma -x c - -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0)
+ifeq ($(HAVE_LZMA), 1)
+LZMA_MSG := ==> building zstd with .xz/.lzma compression support
+LZMACPP = -DZSTD_LZMACOMPRESS -DZSTD_LZMADECOMPRESS
+LZMALD = -llzma
+else
+LZMA_MSG := $(NO_LZMA_MSG)
+endif
+
+
+.PHONY: default all clean clean_decomp_o install uninstall generate_res
+
+default: zstd-release
+
+all: zstd
+
+$(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)
+
+zstd : CPPFLAGS += $(ZLIBCPP)
+zstd : LDFLAGS += $(ZLIBLD)
+zstd : LZMA_MSG := $(NO_LZMA_MSG)
+zstd-nogz : ZLIB_MSG := $(NO_ZLIB_MSG)
+zstd-nogz : LZMA_MSG := $(NO_LZMA_MSG)
+xzstd : CPPFLAGS += $(ZLIBCPP) $(LZMACPP)
+xzstd : LDFLAGS += $(ZLIBLD) $(LZMALD)
+zstd zstd-nogz xzstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
+zstd zstd-nogz xzstd : $(ZSTDLIB_OBJ) zstdcli.o fileio.o bench.o datagen.o dibio.o
+ @echo "$(ZLIB_MSG)"
+ @echo "$(LZMA_MSG)"
+ifneq (,$(filter Windows%,$(OS)))
+ windres/generate_res.bat
+endif
+ $(CC) $(FLAGS) $^ $(RES_FILE) -o zstd$(EXT) $(LDFLAGS)
+
+zstd-release: DEBUGFLAGS :=
+zstd-release: zstd
+
+zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
+zstd32 : $(ZSTDLIB_FILES) zstdcli.c fileio.c bench.c datagen.c dibio.c
+ifneq (,$(filter Windows%,$(OS)))
+ windres/generate_res.bat
+endif
+ $(CC) -m32 $(FLAGS) $^ $(RES32_FILE) -o $@$(EXT)
+
+
+zstd-nolegacy : clean_decomp_o
+ $(MAKE) zstd ZSTD_LEGACY_SUPPORT=0
+
+zstd-pgo : MOREFLAGS = -fprofile-generate
+zstd-pgo : clean zstd
+ ./zstd -b19i1 $(PROFILE_WITH)
+ ./zstd -b16i1 $(PROFILE_WITH)
+ ./zstd -b9i2 $(PROFILE_WITH)
+ ./zstd -b $(PROFILE_WITH)
+ ./zstd -b7i2 $(PROFILE_WITH)
+ ./zstd -b5 $(PROFILE_WITH)
+ $(RM) zstd
+ $(RM) $(ZSTDDECOMP_O)
+ $(MAKE) zstd MOREFLAGS=-fprofile-use
+
+zstd-frugal: $(ZSTD_FILES) zstdcli.c fileio.c
+ $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT $^ -o zstd$(EXT)
+
+zstd-small:
+ CFLAGS="-Os -s" $(MAKE) zstd-frugal
+
+zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c fileio.c
+ $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS $^ -o $@$(EXT)
+
+zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c fileio.c
+ $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT)
+
+zstdmt: CPPFLAGS += -DZSTD_MULTITHREAD
+ifeq (,$(filter Windows%,$(OS)))
+zstdmt: LDFLAGS += -lpthread
+endif
+zstdmt: zstd
+
+generate_res:
+ windres/generate_res.bat
+
+clean:
+ $(MAKE) -C $(ZSTDDIR) clean
+ @$(RM) $(ZSTDDIR)/decompress/*.o $(ZSTDDIR)/decompress/zstd_decompress.gcda
+ @$(RM) core *.o tmp* result* *.gcda dictionary *.zst \
+ zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \
+ *.gcda default.profraw have_zlib$(EXT)
+ @echo Cleaning completed
+
+clean_decomp_o:
+ @$(RM) $(ZSTDDECOMP_O)
+
+
+#-----------------------------------------------------------------------------
+# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
+#-----------------------------------------------------------------------------
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
+
+ifneq (,$(filter $(shell uname),SunOS))
+INSTALL ?= ginstall
+else
+INSTALL ?= install
+endif
+
+PREFIX ?= /usr/local
+DESTDIR ?=
+BINDIR ?= $(PREFIX)/bin
+
+ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS))
+MANDIR ?= $(PREFIX)/man/man1
+else
+MANDIR ?= $(PREFIX)/share/man/man1
+endif
+
+INSTALL_PROGRAM ?= $(INSTALL) -m 755
+INSTALL_SCRIPT ?= $(INSTALL) -m 755
+INSTALL_MAN ?= $(INSTALL) -m 644
+
+install: zstd
+ @echo Installing binaries
+ @$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/
+ @$(INSTALL_PROGRAM) zstd $(DESTDIR)$(BINDIR)/zstd
+ @ln -sf zstd $(DESTDIR)$(BINDIR)/zstdcat
+ @ln -sf zstd $(DESTDIR)$(BINDIR)/unzstd
+ @$(INSTALL_SCRIPT) zstdless $(DESTDIR)$(BINDIR)/zstdless
+ @$(INSTALL_SCRIPT) zstdgrep $(DESTDIR)$(BINDIR)/zstdgrep
+ @echo Installing man pages
+ @$(INSTALL_MAN) zstd.1 $(DESTDIR)$(MANDIR)/zstd.1
+ @ln -sf zstd.1 $(DESTDIR)$(MANDIR)/zstdcat.1
+ @ln -sf zstd.1 $(DESTDIR)$(MANDIR)/unzstd.1
+ @echo zstd installation completed
+
+uninstall:
+ @$(RM) $(DESTDIR)$(BINDIR)/zstdgrep
+ @$(RM) $(DESTDIR)$(BINDIR)/zstdless
+ @$(RM) $(DESTDIR)$(BINDIR)/zstdcat
+ @$(RM) $(DESTDIR)$(BINDIR)/unzstd
+ @$(RM) $(DESTDIR)$(BINDIR)/zstd
+ @$(RM) $(DESTDIR)$(MANDIR)/zstdcat.1
+ @$(RM) $(DESTDIR)$(MANDIR)/unzstd.1
+ @$(RM) $(DESTDIR)$(MANDIR)/zstd.1
+ @echo zstd programs successfully uninstalled
+endif
diff --git a/programs/README.md b/programs/README.md
new file mode 100644
index 000000000000..203fd7b49bce
--- /dev/null
+++ b/programs/README.md
@@ -0,0 +1,94 @@
+Command Line Interface for Zstandard library
+============================================
+
+Command Line Interface (CLI) can be created using the `make` command without any additional parameters.
+There are however other Makefile targets that create different variations of CLI:
+- `zstd` : default CLI supporting gzip-like arguments; includes dictionary builder, benchmark, and support for decompression of legacy zstd versions
+- `zstd32` : Same as `zstd`, but forced to compile in 32-bits mode
+- `zstd_nolegacy` : Same as `zstd` except of support for decompression of legacy zstd versions
+- `zstd-small` : CLI optimized for minimal size; without dictionary builder, benchmark, and support for decompression of legacy zstd versions
+- `zstd-compress` : compressor-only version of CLI; without dictionary builder, benchmark, and support for decompression of legacy zstd versions
+- `zstd-decompress` : decompressor-only version of CLI; without dictionary builder, benchmark, and support for decompression of legacy zstd versions
+
+
+#### Aggregation of parameters
+CLI supports aggregation of parameters i.e. `-b1`, `-e18`, and `-i1` can be joined into `-b1e18i1`.
+
+
+#### Dictionary builder in Command Line Interface
+Zstd offers a training mode, which can be used to tune the algorithm for a selected
+type of data, by providing it with a few samples. The result of the training is stored
+in a file selected with the `-o` option (default name is `dictionary`),
+which can be loaded before compression and decompression.
+
+Using a dictionary, the compression ratio achievable on small data improves dramatically.
+These compression gains are achieved while simultaneously providing faster compression and decompression speeds.
+Dictionary work if there is some correlation in a family of small data (there is no universal dictionary).
+Hence, deploying one dictionary per type of data will provide the greater benefits.
+Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm
+will rely more and more on previously decoded content to compress the rest of the file.
+
+Usage of the dictionary builder and created dictionaries with CLI:
+
+1. Create the dictionary : `zstd --train FullPathToTrainingSet/* -o dictionaryName`
+2. Compress with the dictionary: `zstd FILE -D dictionaryName`
+3. Decompress with the dictionary: `zstd --decompress FILE.zst -D dictionaryName`
+
+
+
+#### Benchmark in Command Line Interface
+CLI includes in-memory compression benchmark module for zstd.
+The benchmark is conducted using given filenames. The files are read into memory and joined together.
+It makes benchmark more precise as it eliminates I/O overhead.
+Many filenames can be supplied as multiple parameters, parameters with wildcards or
+names of directories can be used as parameters with the `-r` option.
+
+The benchmark measures ratio, compressed size, compression and decompression speed.
+One can select compression levels starting from `-b` and ending with `-e`.
+The `-i` parameter selects minimal time used for each of tested levels.
+
+
+
+#### Usage of Command Line Interface
+The full list of options can be obtained with `-h` or `-H` parameter:
+```
+Usage :
+ zstd [args] [FILE(s)] [-o file]
+
+FILE : a filename
+ with no FILE, or when FILE is - , read standard input
+Arguments :
+ -# : # compression level (1-19, default:3)
+ -d : decompression
+ -D file: use `file` as Dictionary
+ -o file: result stored into `file` (only if 1 input file)
+ -f : overwrite output without prompting
+--rm : remove source file(s) after successful de/compression
+ -k : preserve source file(s) (default)
+ -h/-H : display help/long help and exit
+
+Advanced arguments :
+ -V : display Version number and exit
+ -v : verbose mode; specify multiple times to increase log level (default:2)
+ -q : suppress warnings; specify twice to suppress errors too
+ -c : force write to standard output, even if it is the console
+ -r : operate recursively on directories
+--ultra : enable levels beyond 19, up to 22 (requires more memory)
+--no-dictID : don't write dictID into header (dictionary compression)
+--[no-]check : integrity check (default:enabled)
+--test : test compressed file integrity
+--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)
+
+Dictionary builder :
+--train ## : create a dictionary from a training set of files
+ -o file : `file` is dictionary name (default: dictionary)
+--maxdict ## : limit dictionary to specified size (default : 112640)
+ -s# : dictionary selectivity level (default: 9)
+--dictID ## : force dictionary ID to specified value (default: random)
+
+Benchmark arguments :
+ -b# : benchmark file(s), using # compression level (default : 1)
+ -e# : test all compression levels from -bX to # (default: 1)
+ -i# : minimum evaluation time in seconds (default : 3s)
+ -B# : cut file into independent blocks of size # (default: no block)
+ ``` \ No newline at end of file
diff --git a/programs/bench.c b/programs/bench.c
new file mode 100644
index 000000000000..2dd1cfb0fab1
--- /dev/null
+++ b/programs/bench.c
@@ -0,0 +1,595 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+
+
+/* **************************************
+* Tuning parameters
+****************************************/
+#ifndef BMK_TIMETEST_DEFAULT_S /* default minimum time per test */
+#define BMK_TIMETEST_DEFAULT_S 3
+#endif
+
+
+/* **************************************
+* Compiler Warnings
+****************************************/
+#ifdef _MSC_VER
+# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
+#endif
+
+
+/* *************************************
+* Includes
+***************************************/
+#include "platform.h" /* Large Files support */
+#include "util.h" /* UTIL_getFileSize, UTIL_sleep */
+#include <stdlib.h> /* malloc, free */
+#include <string.h> /* memset */
+#include <stdio.h> /* fprintf, fopen */
+#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
+
+#include "mem.h"
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+#include "datagen.h" /* RDG_genBuffer */
+#include "xxhash.h"
+#include "zstdmt_compress.h"
+
+
+/* *************************************
+* Constants
+***************************************/
+#ifndef ZSTD_GIT_COMMIT
+# define ZSTD_GIT_COMMIT_STRING ""
+#else
+# define ZSTD_GIT_COMMIT_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_GIT_COMMIT)
+#endif
+
+#define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */
+#define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */
+#define COOLPERIOD_SEC 10
+
+#define KB *(1 <<10)
+#define MB *(1 <<20)
+#define GB *(1U<<30)
+
+static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
+
+static U32 g_compressibilityDefault = 50;
+
+
+/* *************************************
+* console display
+***************************************/
+#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
+
+#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
+ if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \
+ { g_time = clock(); DISPLAY(__VA_ARGS__); \
+ if (g_displayLevel>=4) fflush(stdout); } }
+static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
+static clock_t g_time = 0;
+
+
+/* *************************************
+* Exceptions
+***************************************/
+#ifndef DEBUG
+# define DEBUG 0
+#endif
+#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
+#define EXM_THROW(error, ...) \
+{ \
+ DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
+ DISPLAYLEVEL(1, "Error %i : ", error); \
+ DISPLAYLEVEL(1, __VA_ARGS__); \
+ DISPLAYLEVEL(1, " \n"); \
+ exit(error); \
+}
+
+
+/* *************************************
+* Benchmark Parameters
+***************************************/
+static int g_additionalParam = 0;
+static U32 g_decodeOnly = 0;
+
+void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
+
+void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }
+
+static U32 g_nbSeconds = BMK_TIMETEST_DEFAULT_S;
+void BMK_setNbSeconds(unsigned nbSeconds)
+{
+ g_nbSeconds = nbSeconds;
+ DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression - \n", g_nbSeconds);
+}
+
+static size_t g_blockSize = 0;
+void BMK_setBlockSize(size_t blockSize)
+{
+ g_blockSize = blockSize;
+ if (g_blockSize) DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10));
+}
+
+void BMK_setDecodeOnlyMode(unsigned decodeFlag) { g_decodeOnly = (decodeFlag>0); }
+
+static U32 g_nbThreads = 1;
+void BMK_setNbThreads(unsigned nbThreads) {
+#ifndef ZSTD_MULTITHREAD
+ if (nbThreads > 1) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n");
+#endif
+ g_nbThreads = nbThreads;
+}
+
+
+/* ********************************************************
+* Bench functions
+**********************************************************/
+typedef struct {
+ const void* srcPtr;
+ size_t srcSize;
+ void* cPtr;
+ size_t cRoom;
+ size_t cSize;
+ void* resPtr;
+ size_t resSize;
+} blockParam_t;
+
+
+#define MIN(a,b) ((a)<(b) ? (a) : (b))
+#define MAX(a,b) ((a)>(b) ? (a) : (b))
+
+static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
+ const char* displayName, int cLevel,
+ const size_t* fileSizes, U32 nbFiles,
+ const void* dictBuffer, size_t dictBufferSize,
+ ZSTD_compressionParameters *comprParams)
+{
+ size_t const blockSize = ((g_blockSize>=32 && !g_decodeOnly) ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
+ size_t const avgSize = MIN(g_blockSize, (srcSize / nbFiles));
+ U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
+ blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
+ size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
+ void* const compressedBuffer = malloc(maxCompressedSize);
+ void* resultBuffer = malloc(srcSize);
+ ZSTDMT_CCtx* const mtctx = ZSTDMT_createCCtx(g_nbThreads);
+ ZSTD_CCtx* const ctx = ZSTD_createCCtx();
+ ZSTD_DCtx* const dctx = ZSTD_createDCtx();
+ size_t const loadedCompressedSize = srcSize;
+ size_t cSize = 0;
+ double ratio = 0.;
+ U32 nbBlocks;
+ UTIL_freq_t ticksPerSecond;
+
+ /* checks */
+ if (!compressedBuffer || !resultBuffer || !blockTable || !ctx || !dctx)
+ EXM_THROW(31, "allocation error : not enough memory");
+
+ /* init */
+ if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */
+ UTIL_initTimer(&ticksPerSecond);
+
+ if (g_decodeOnly) {
+ const char* srcPtr = (const char*) srcBuffer;
+ U64 dSize64 = 0;
+ U32 fileNb;
+ for (fileNb=0; fileNb<nbFiles; fileNb++) {
+ U64 const fSize64 = ZSTD_findDecompressedSize(srcPtr, fileSizes[fileNb]);
+ if (fSize64==0) EXM_THROW(32, "Impossible to determine original size ");
+ dSize64 += fSize64;
+ srcPtr += fileSizes[fileNb];
+ }
+ { size_t const decodedSize = (size_t)dSize64;
+ if (dSize64 > decodedSize) EXM_THROW(32, "original size is too large");
+ if (decodedSize==0) EXM_THROW(32, "Impossible to determine original size ");
+ free(resultBuffer);
+ resultBuffer = malloc(decodedSize);
+ if (!resultBuffer) EXM_THROW(33, "not enough memory");
+ cSize = srcSize;
+ srcSize = decodedSize;
+ ratio = (double)srcSize / (double)cSize;
+ } }
+
+ /* Init blockTable data */
+ { const char* srcPtr = (const char*)srcBuffer;
+ char* cPtr = (char*)compressedBuffer;
+ char* resPtr = (char*)resultBuffer;
+ U32 fileNb;
+ for (nbBlocks=0, fileNb=0; fileNb<nbFiles; fileNb++) {
+ size_t remaining = fileSizes[fileNb];
+ U32 const nbBlocksforThisFile = g_decodeOnly ? 1 : (U32)((remaining + (blockSize-1)) / blockSize);
+ U32 const blockEnd = nbBlocks + nbBlocksforThisFile;
+ for ( ; nbBlocks<blockEnd; nbBlocks++) {
+ size_t const thisBlockSize = MIN(remaining, blockSize);
+ blockTable[nbBlocks].srcPtr = (const void*)srcPtr;
+ blockTable[nbBlocks].srcSize = thisBlockSize;
+ blockTable[nbBlocks].cPtr = (void*)cPtr;
+ blockTable[nbBlocks].cRoom = g_decodeOnly ? thisBlockSize : ZSTD_compressBound(thisBlockSize);
+ blockTable[nbBlocks].cSize = blockTable[nbBlocks].cRoom;
+ blockTable[nbBlocks].resPtr = (void*)resPtr;
+ blockTable[nbBlocks].resSize = g_decodeOnly ? (size_t) ZSTD_findDecompressedSize(srcPtr, thisBlockSize) : thisBlockSize;
+ srcPtr += thisBlockSize;
+ cPtr += blockTable[nbBlocks].cRoom;
+ resPtr += thisBlockSize;
+ remaining -= thisBlockSize;
+ } } }
+
+ /* warmimg up memory */
+ RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
+
+ /* Bench */
+ { U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL);
+ U64 const crcOrig = g_decodeOnly ? 0 : XXH64(srcBuffer, srcSize, 0);
+ UTIL_time_t coolTime;
+ U64 const maxTime = (g_nbSeconds * TIMELOOP_MICROSEC) + 1;
+ U64 totalCTime=0, totalDTime=0;
+ U32 cCompleted=g_decodeOnly, dCompleted=0;
+# define NB_MARKS 4
+ const char* const marks[NB_MARKS] = { " |", " /", " =", "\\" };
+ U32 markNb = 0;
+
+ UTIL_getTime(&coolTime);
+ DISPLAYLEVEL(2, "\r%79s\r", "");
+ while (!cCompleted || !dCompleted) {
+
+ /* overheat protection */
+ if (UTIL_clockSpanMicro(coolTime, ticksPerSecond) > ACTIVEPERIOD_MICROSEC) {
+ DISPLAYLEVEL(2, "\rcooling down ... \r");
+ UTIL_sleep(COOLPERIOD_SEC);
+ UTIL_getTime(&coolTime);
+ }
+
+ if (!g_decodeOnly) {
+ UTIL_time_t clockStart;
+ /* Compression */
+ DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize);
+ if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */
+
+ UTIL_sleepMilli(1); /* give processor time to other processes */
+ UTIL_waitForNextTick(ticksPerSecond);
+ UTIL_getTime(&clockStart);
+
+ if (!cCompleted) { /* still some time to do compression tests */
+ ZSTD_parameters zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
+ ZSTD_customMem const cmem = { NULL, NULL, NULL };
+ U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
+ U32 nbLoops = 0;
+ ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1, zparams, cmem);
+ if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
+ if (comprParams->windowLog) zparams.cParams.windowLog = comprParams->windowLog;
+ if (comprParams->chainLog) zparams.cParams.chainLog = comprParams->chainLog;
+ if (comprParams->hashLog) zparams.cParams.hashLog = comprParams->hashLog;
+ if (comprParams->searchLog) zparams.cParams.searchLog = comprParams->searchLog;
+ if (comprParams->searchLength) zparams.cParams.searchLength = comprParams->searchLength;
+ if (comprParams->targetLength) zparams.cParams.targetLength = comprParams->targetLength;
+ if (comprParams->strategy) zparams.cParams.strategy = (ZSTD_strategy)(comprParams->strategy - 1);
+ do {
+ U32 blockNb;
+ size_t rSize;
+ for (blockNb=0; blockNb<nbBlocks; blockNb++) {
+ if (dictBufferSize) {
+ rSize = ZSTD_compress_usingCDict(ctx,
+ blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
+ blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
+ cdict);
+ } else {
+#ifdef ZSTD_MULTITHREAD /* note : limitation : MT single-pass does not support compression with dictionary */
+ rSize = ZSTDMT_compressCCtx(mtctx,
+ blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
+ blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
+ cLevel);
+#else
+ rSize = ZSTD_compress_advanced (ctx,
+ blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
+ blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, NULL, 0, zparams);
+#endif
+ }
+ if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingCDict() failed : %s", ZSTD_getErrorName(rSize));
+ blockTable[blockNb].cSize = rSize;
+ }
+ nbLoops++;
+ } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
+ ZSTD_freeCDict(cdict);
+ { U64 const clockSpanMicro = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
+ if (clockSpanMicro < fastestC*nbLoops) fastestC = clockSpanMicro / nbLoops;
+ totalCTime += clockSpanMicro;
+ cCompleted = (totalCTime >= maxTime);
+ } }
+
+ cSize = 0;
+ { U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
+ ratio = (double)srcSize / (double)cSize;
+ markNb = (markNb+1) % NB_MARKS;
+ DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
+ marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
+ (double)srcSize / fastestC );
+ } else { /* g_decodeOnly */
+ memcpy(compressedBuffer, srcBuffer, loadedCompressedSize);
+ }
+
+#if 0 /* disable decompression test */
+ dCompleted=1;
+ (void)totalDTime; (void)fastestD; (void)crcOrig; /* unused when decompression disabled */
+#else
+ /* Decompression */
+ if (!dCompleted) memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */
+
+ UTIL_sleepMilli(1); /* give processor time to other processes */
+ UTIL_waitForNextTick(ticksPerSecond);
+
+ if (!dCompleted) {
+ U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
+ U32 nbLoops = 0;
+ UTIL_time_t clockStart;
+ ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictBufferSize);
+ if (!ddict) EXM_THROW(2, "ZSTD_createDDict() allocation failure");
+ UTIL_getTime(&clockStart);
+ do {
+ U32 blockNb;
+ for (blockNb=0; blockNb<nbBlocks; blockNb++) {
+ size_t const regenSize = ZSTD_decompress_usingDDict(dctx,
+ blockTable[blockNb].resPtr, blockTable[blockNb].resSize,
+ blockTable[blockNb].cPtr, blockTable[blockNb].cSize,
+ ddict);
+ if (ZSTD_isError(regenSize)) {
+ DISPLAY("ZSTD_decompress_usingDDict() failed on block %u of size %u : %s \n",
+ blockNb, (U32)blockTable[blockNb].cSize, ZSTD_getErrorName(regenSize));
+ clockLoop = 0; /* force immediate test end */
+ break;
+ }
+ blockTable[blockNb].resSize = regenSize;
+ }
+ nbLoops++;
+ } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
+ ZSTD_freeDDict(ddict);
+ { U64 const clockSpanMicro = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
+ if (clockSpanMicro < fastestD*nbLoops) fastestD = clockSpanMicro / nbLoops;
+ totalDTime += clockSpanMicro;
+ dCompleted = (totalDTime >= maxTime);
+ } }
+
+ markNb = (markNb+1) % NB_MARKS;
+ DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
+ marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
+ (double)srcSize / fastestC,
+ (double)srcSize / fastestD );
+
+ /* CRC Checking */
+ { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
+ if (!g_decodeOnly && (crcOrig!=crcCheck)) {
+ size_t u;
+ DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
+ for (u=0; u<srcSize; u++) {
+ if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u]) {
+ U32 segNb, bNb, pos;
+ size_t bacc = 0;
+ DISPLAY("Decoding error at pos %u ", (U32)u);
+ for (segNb = 0; segNb < nbBlocks; segNb++) {
+ if (bacc + blockTable[segNb].srcSize > u) break;
+ bacc += blockTable[segNb].srcSize;
+ }
+ pos = (U32)(u - bacc);
+ bNb = pos / (128 KB);
+ DISPLAY("(block %u, sub %u, pos %u) \n", segNb, bNb, pos);
+ if (u>5) {
+ int n;
+ for (n=-5; n<0; n++) DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]);
+ DISPLAY(" :%02X: ", ((const BYTE*)srcBuffer)[u]);
+ for (n=1; n<3; n++) DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]);
+ DISPLAY(" \n");
+ for (n=-5; n<0; n++) DISPLAY("%02X ", ((const BYTE*)resultBuffer)[u+n]);
+ DISPLAY(" :%02X: ", ((const BYTE*)resultBuffer)[u]);
+ for (n=1; n<3; n++) DISPLAY("%02X ", ((const BYTE*)resultBuffer)[u+n]);
+ DISPLAY(" \n");
+ }
+ break;
+ }
+ if (u==srcSize-1) { /* should never happen */
+ DISPLAY("no difference detected\n");
+ } }
+ break;
+ } } /* CRC Checking */
+#endif
+ } /* for (testNb = 1; testNb <= (g_nbSeconds + !g_nbSeconds); testNb++) */
+
+ if (g_displayLevel == 1) {
+ double cSpeed = (double)srcSize / fastestC;
+ double dSpeed = (double)srcSize / fastestD;
+ if (g_additionalParam)
+ DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam);
+ else
+ DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName);
+ }
+ DISPLAYLEVEL(2, "%2i#\n", cLevel);
+ } /* Bench */
+
+ /* clean up */
+ free(blockTable);
+ free(compressedBuffer);
+ free(resultBuffer);
+ ZSTDMT_freeCCtx(mtctx);
+ ZSTD_freeCCtx(ctx);
+ ZSTD_freeDCtx(dctx);
+ return 0;
+}
+
+
+static size_t BMK_findMaxMem(U64 requiredMem)
+{
+ size_t const step = 64 MB;
+ BYTE* testmem = NULL;
+
+ requiredMem = (((requiredMem >> 26) + 1) << 26);
+ requiredMem += step;
+ if (requiredMem > maxMemory) requiredMem = maxMemory;
+
+ do {
+ testmem = (BYTE*)malloc((size_t)requiredMem);
+ requiredMem -= step;
+ } while (!testmem);
+
+ free(testmem);
+ return (size_t)(requiredMem);
+}
+
+static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
+ const char* displayName, int cLevel, int cLevelLast,
+ const size_t* fileSizes, unsigned nbFiles,
+ const void* dictBuffer, size_t dictBufferSize,
+ ZSTD_compressionParameters *compressionParams, int setRealTimePrio)
+{
+ int l;
+
+ const char* pch = strrchr(displayName, '\\'); /* Windows */
+ if (!pch) pch = strrchr(displayName, '/'); /* Linux */
+ if (pch) displayName = pch+1;
+
+ if (setRealTimePrio) {
+ DISPLAYLEVEL(2, "Note : switching to a real-time priority \n");
+ SET_REALTIME_PRIORITY;
+ }
+
+ if (g_displayLevel == 1 && !g_additionalParam)
+ DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10));
+
+ if (cLevelLast < cLevel) cLevelLast = cLevel;
+
+ for (l=cLevel; l <= cLevelLast; l++) {
+ BMK_benchMem(srcBuffer, benchedSize,
+ displayName, l,
+ fileSizes, nbFiles,
+ dictBuffer, dictBufferSize, compressionParams);
+ }
+}
+
+
+/*! BMK_loadFiles() :
+ Loads `buffer` with content of files listed within `fileNamesTable`.
+ At most, fills `buffer` entirely */
+static void BMK_loadFiles(void* buffer, size_t bufferSize,
+ size_t* fileSizes,
+ const char** fileNamesTable, unsigned nbFiles)
+{
+ size_t pos = 0, totalSize = 0;
+ unsigned n;
+ for (n=0; n<nbFiles; n++) {
+ FILE* f;
+ U64 fileSize = UTIL_getFileSize(fileNamesTable[n]);
+ if (UTIL_isDirectory(fileNamesTable[n])) {
+ DISPLAYLEVEL(2, "Ignoring %s directory... \n", fileNamesTable[n]);
+ fileSizes[n] = 0;
+ continue;
+ }
+ f = fopen(fileNamesTable[n], "rb");
+ if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
+ DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
+ if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */
+ { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
+ if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
+ pos += readSize; }
+ fileSizes[n] = (size_t)fileSize;
+ totalSize += (size_t)fileSize;
+ fclose(f);
+ }
+
+ if (totalSize == 0) EXM_THROW(12, "no data to bench");
+}
+
+static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel,
+ int cLevelLast, ZSTD_compressionParameters *compressionParams, int setRealTimePrio)
+{
+ void* srcBuffer;
+ size_t benchedSize;
+ void* dictBuffer = NULL;
+ size_t dictBufferSize = 0;
+ size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
+ U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
+ char mfName[20] = {0};
+
+ if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes");
+
+ /* Load dictionary */
+ if (dictFileName != NULL) {
+ U64 dictFileSize = UTIL_getFileSize(dictFileName);
+ if (dictFileSize > 64 MB) EXM_THROW(10, "dictionary file %s too large", dictFileName);
+ dictBufferSize = (size_t)dictFileSize;
+ dictBuffer = malloc(dictBufferSize);
+ if (dictBuffer==NULL) EXM_THROW(11, "not enough memory for dictionary (%u bytes)", (U32)dictBufferSize);
+ BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1);
+ }
+
+ /* Memory allocation & restrictions */
+ benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3;
+ if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
+ if (benchedSize < totalSizeToLoad)
+ DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
+ srcBuffer = malloc(benchedSize);
+ if (!srcBuffer) EXM_THROW(12, "not enough memory");
+
+ /* Load input buffer */
+ BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
+
+ /* Bench */
+ snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
+ { const char* displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
+ BMK_benchCLevel(srcBuffer, benchedSize,
+ displayName, cLevel, cLevelLast,
+ fileSizes, nbFiles,
+ dictBuffer, dictBufferSize, compressionParams, setRealTimePrio);
+ }
+
+ /* clean up */
+ free(srcBuffer);
+ free(dictBuffer);
+ free(fileSizes);
+}
+
+
+static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, ZSTD_compressionParameters* compressionParams, int setRealTimePrio)
+{
+ char name[20] = {0};
+ size_t benchedSize = 10000000;
+ void* const srcBuffer = malloc(benchedSize);
+
+ /* Memory allocation */
+ if (!srcBuffer) EXM_THROW(21, "not enough memory");
+
+ /* Fill input buffer */
+ RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
+
+ /* Bench */
+ snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
+ BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams, setRealTimePrio);
+
+ /* clean up */
+ free(srcBuffer);
+}
+
+
+int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
+ int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams, int setRealTimePrio)
+{
+ double const compressibility = (double)g_compressibilityDefault / 100;
+
+ if (cLevel < 1) cLevel = 1; /* minimum compression level */
+ if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
+ if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel();
+ if (cLevelLast < cLevel) cLevelLast = cLevel;
+ if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
+
+ if (nbFiles == 0)
+ BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams, setRealTimePrio);
+ else
+ BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams, setRealTimePrio);
+ return 0;
+}
diff --git a/programs/bench.h b/programs/bench.h
new file mode 100644
index 000000000000..77a527f8ff82
--- /dev/null
+++ b/programs/bench.h
@@ -0,0 +1,29 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+
+#ifndef BENCH_H_121279284357
+#define BENCH_H_121279284357
+
+#include <stddef.h> /* size_t */
+#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
+#include "zstd.h" /* ZSTD_compressionParameters */
+
+int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,const char* dictFileName,
+ int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams, int setRealTimePrio);
+
+/* Set Parameters */
+void BMK_setNbSeconds(unsigned nbLoops);
+void BMK_setBlockSize(size_t blockSize);
+void BMK_setNbThreads(unsigned nbThreads);
+void BMK_setNotificationLevel(unsigned level);
+void BMK_setAdditionalParam(int additionalParam);
+void BMK_setDecodeOnlyMode(unsigned decodeFlag);
+
+#endif /* BENCH_H_121279284357 */
diff --git a/programs/datagen.c b/programs/datagen.c
new file mode 100644
index 000000000000..d0116b97232f
--- /dev/null
+++ b/programs/datagen.c
@@ -0,0 +1,180 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+
+
+/*-************************************
+* Dependencies
+**************************************/
+#include "platform.h" /* SET_BINARY_MODE */
+#include <stdlib.h> /* malloc, free */
+#include <stdio.h> /* FILE, fwrite, fprintf */
+#include <string.h> /* memcpy */
+#include "mem.h" /* U32 */
+
+
+/*-************************************
+* Macros
+**************************************/
+#define KB *(1 <<10)
+#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
+
+#define RDG_DEBUG 0
+#define TRACE(...) if (RDG_DEBUG) fprintf(stderr, __VA_ARGS__ )
+
+
+/*-************************************
+* Local constants
+**************************************/
+#define LTLOG 13
+#define LTSIZE (1<<LTLOG)
+#define LTMASK (LTSIZE-1)
+
+
+/*-*******************************************************
+* Local Functions
+*********************************************************/
+#define RDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+static U32 RDG_rand(U32* src)
+{
+ static const U32 prime1 = 2654435761U;
+ static const U32 prime2 = 2246822519U;
+ U32 rand32 = *src;
+ rand32 *= prime1;
+ rand32 ^= prime2;
+ rand32 = RDG_rotl32(rand32, 13);
+ *src = rand32;
+ return rand32 >> 5;
+}
+
+
+static void RDG_fillLiteralDistrib(BYTE* ldt, double ld)
+{
+ BYTE const firstChar = (ld<=0.0) ? 0 : '(';
+ BYTE const lastChar = (ld<=0.0) ? 255 : '}';
+ BYTE character = (ld<=0.0) ? 0 : '0';
+ U32 u;
+
+ if (ld<=0.0) ld = 0.0;
+ for (u=0; u<LTSIZE; ) {
+ U32 const weight = (U32)((double)(LTSIZE - u) * ld) + 1;
+ U32 const end = MIN ( u + weight , LTSIZE);
+ while (u < end) ldt[u++] = character;
+ character++;
+ if (character > lastChar) character = firstChar;
+ }
+}
+
+
+static BYTE RDG_genChar(U32* seed, const BYTE* ldt)
+{
+ U32 const id = RDG_rand(seed) & LTMASK;
+ return ldt[id]; /* memory-sanitizer fails here, stating "uninitialized value" when table initialized with P==0.0. Checked : table is fully initialized */
+}
+
+
+static U32 RDG_rand15Bits (unsigned* seedPtr)
+{
+ return RDG_rand(seedPtr) & 0x7FFF;
+}
+
+static U32 RDG_randLength(unsigned* seedPtr)
+{
+ if (RDG_rand(seedPtr) & 7) return (RDG_rand(seedPtr) & 0xF); /* small length */
+ return (RDG_rand(seedPtr) & 0x1FF) + 0xF;
+}
+
+void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, const BYTE* ldt, unsigned* seedPtr)
+{
+ BYTE* const buffPtr = (BYTE*)buffer;
+ U32 const matchProba32 = (U32)(32768 * matchProba);
+ size_t pos = prefixSize;
+ U32 prevOffset = 1;
+
+ /* special case : sparse content */
+ while (matchProba >= 1.0) {
+ size_t size0 = RDG_rand(seedPtr) & 3;
+ size0 = (size_t)1 << (16 + size0 * 2);
+ size0 += RDG_rand(seedPtr) & (size0-1); /* because size0 is power of 2*/
+ if (buffSize < pos + size0) {
+ memset(buffPtr+pos, 0, buffSize-pos);
+ return;
+ }
+ memset(buffPtr+pos, 0, size0);
+ pos += size0;
+ buffPtr[pos-1] = RDG_genChar(seedPtr, ldt);
+ continue;
+ }
+
+ /* init */
+ if (pos==0) buffPtr[0] = RDG_genChar(seedPtr, ldt), pos=1;
+
+ /* Generate compressible data */
+ while (pos < buffSize) {
+ /* Select : Literal (char) or Match (within 32K) */
+ if (RDG_rand15Bits(seedPtr) < matchProba32) {
+ /* Copy (within 32K) */
+ U32 const length = RDG_randLength(seedPtr) + 4;
+ U32 const d = (U32) MIN(pos + length , buffSize);
+ U32 const repeatOffset = (RDG_rand(seedPtr) & 15) == 2;
+ U32 const randOffset = RDG_rand15Bits(seedPtr) + 1;
+ U32 const offset = repeatOffset ? prevOffset : (U32) MIN(randOffset , pos);
+ size_t match = pos - offset;
+ while (pos < d) buffPtr[pos++] = buffPtr[match++]; /* correctly manages overlaps */
+ prevOffset = offset;
+ } else {
+ /* Literal (noise) */
+ U32 const length = RDG_randLength(seedPtr);
+ U32 const d = (U32) MIN(pos + length, buffSize);
+ while (pos < d) buffPtr[pos++] = RDG_genChar(seedPtr, ldt);
+ } }
+}
+
+
+void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed)
+{
+ BYTE ldt[LTSIZE];
+ memset(ldt, '0', sizeof(ldt)); /* yes, character '0', this is intentional */
+ if (litProba<=0.0) litProba = matchProba / 4.5;
+ RDG_fillLiteralDistrib(ldt, litProba);
+ RDG_genBlock(buffer, size, 0, matchProba, ldt, &seed);
+}
+
+
+void RDG_genStdout(unsigned long long size, double matchProba, double litProba, unsigned seed)
+{
+ size_t const stdBlockSize = 128 KB;
+ size_t const stdDictSize = 32 KB;
+ BYTE* const buff = (BYTE*)malloc(stdDictSize + stdBlockSize);
+ U64 total = 0;
+ BYTE ldt[LTSIZE]; /* literals distribution table */
+
+ /* init */
+ if (buff==NULL) { perror("datagen"); exit(1); }
+ if (litProba<=0.0) litProba = matchProba / 4.5;
+ memset(ldt, '0', sizeof(ldt)); /* yes, character '0', this is intentional */
+ RDG_fillLiteralDistrib(ldt, litProba);
+ SET_BINARY_MODE(stdout);
+
+ /* Generate initial dict */
+ RDG_genBlock(buff, stdDictSize, 0, matchProba, ldt, &seed);
+
+ /* Generate compressible data */
+ while (total < size) {
+ size_t const genBlockSize = (size_t) (MIN (stdBlockSize, size-total));
+ RDG_genBlock(buff, stdDictSize+stdBlockSize, stdDictSize, matchProba, ldt, &seed);
+ total += genBlockSize;
+ { size_t const unused = fwrite(buff, 1, genBlockSize, stdout); (void)unused; }
+ /* update dict */
+ memcpy(buff, buff + stdBlockSize, stdDictSize);
+ }
+
+ /* cleanup */
+ free(buff);
+}
diff --git a/programs/datagen.h b/programs/datagen.h
new file mode 100644
index 000000000000..094056b696ca
--- /dev/null
+++ b/programs/datagen.h
@@ -0,0 +1,27 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+#ifndef DATAGEN_H
+#define DATAGEN_H
+
+#include <stddef.h> /* size_t */
+
+void RDG_genStdout(unsigned long long size, double matchProba, double litProba, unsigned seed);
+void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed);
+/*!RDG_genBuffer
+ Generate 'size' bytes of compressible data into 'buffer'.
+ Compressibility can be controlled using 'matchProba', which is floating point value between 0 and 1.
+ 'LitProba' is optional, it affect variability of individual bytes. If litProba==0.0, default value will be used.
+ Generated data pattern can be modified using different 'seed'.
+ For a triplet (matchProba, litProba, seed), the function always generate the same content.
+
+ RDG_genStdout
+ Same as RDG_genBuffer, but generates data into stdout
+*/
+
+#endif
diff --git a/programs/dibio.c b/programs/dibio.c
new file mode 100644
index 000000000000..5ef202c8abad
--- /dev/null
+++ b/programs/dibio.c
@@ -0,0 +1,306 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+
+
+/* **************************************
+* Compiler Warnings
+****************************************/
+#ifdef _MSC_VER
+# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
+#endif
+
+
+/*-*************************************
+* Includes
+***************************************/
+#include "platform.h" /* Large Files support */
+#include "util.h" /* UTIL_getFileSize, UTIL_getTotalFileSize */
+#include <stdlib.h> /* malloc, free */
+#include <string.h> /* memset */
+#include <stdio.h> /* fprintf, fopen, ftello64 */
+#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
+#include <errno.h> /* errno */
+
+#include "mem.h" /* read */
+#include "error_private.h"
+#include "dibio.h"
+
+
+/*-*************************************
+* Constants
+***************************************/
+#define KB *(1 <<10)
+#define MB *(1 <<20)
+#define GB *(1U<<30)
+
+#define SAMPLESIZE_MAX (128 KB)
+#define MEMMULT 11 /* rough estimation : memory cost to analyze 1 byte of sample */
+#define COVER_MEMMULT 9 /* rough estimation : memory cost to analyze 1 byte of sample */
+static const size_t maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t));
+
+#define NOISELENGTH 32
+
+
+/*-*************************************
+* Console display
+***************************************/
+#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+static unsigned g_displayLevel = 0; /* 0 : no display; 1: errors; 2: default; 4: full information */
+
+#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
+ if ((DIB_clockSpan(g_time) > refreshRate) || (g_displayLevel>=4)) \
+ { g_time = clock(); DISPLAY(__VA_ARGS__); \
+ if (g_displayLevel>=4) fflush(stdout); } }
+static const clock_t refreshRate = CLOCKS_PER_SEC * 2 / 10;
+static clock_t g_time = 0;
+
+static clock_t DIB_clockSpan(clock_t nPrevious) { return clock() - nPrevious; }
+
+
+/*-*************************************
+* Exceptions
+***************************************/
+#ifndef DEBUG
+# define DEBUG 0
+#endif
+#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
+#define EXM_THROW(error, ...) \
+{ \
+ DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
+ DISPLAYLEVEL(1, "Error %i : ", error); \
+ DISPLAYLEVEL(1, __VA_ARGS__); \
+ DISPLAYLEVEL(1, "\n"); \
+ exit(error); \
+}
+
+
+/* ********************************************************
+* Helper functions
+**********************************************************/
+unsigned DiB_isError(size_t errorCode) { return ERR_isError(errorCode); }
+
+const char* DiB_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
+
+#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
+
+
+/* ********************************************************
+* File related operations
+**********************************************************/
+/** DiB_loadFiles() :
+* @return : nb of files effectively loaded into `buffer` */
+static unsigned DiB_loadFiles(void* buffer, size_t* bufferSizePtr,
+ size_t* fileSizes,
+ const char** fileNamesTable, unsigned nbFiles)
+{
+ char* const buff = (char*)buffer;
+ size_t pos = 0;
+ unsigned n;
+
+ for (n=0; n<nbFiles; n++) {
+ const char* const fileName = fileNamesTable[n];
+ unsigned long long const fs64 = UTIL_getFileSize(fileName);
+ size_t const fileSize = (size_t) MIN(fs64, SAMPLESIZE_MAX);
+ if (fileSize > *bufferSizePtr-pos) break;
+ { FILE* const f = fopen(fileName, "rb");
+ if (f==NULL) EXM_THROW(10, "zstd: dictBuilder: %s %s ", fileName, strerror(errno));
+ DISPLAYUPDATE(2, "Loading %s... \r", fileName);
+ { size_t const readSize = fread(buff+pos, 1, fileSize, f);
+ if (readSize != fileSize) EXM_THROW(11, "Pb reading %s", fileName);
+ pos += readSize; }
+ fileSizes[n] = fileSize;
+ fclose(f);
+ } }
+ DISPLAYLEVEL(2, "\r%79s\r", "");
+ *bufferSizePtr = pos;
+ return n;
+}
+
+#define DiB_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+static U32 DiB_rand(U32* src)
+{
+ static const U32 prime1 = 2654435761U;
+ static const U32 prime2 = 2246822519U;
+ U32 rand32 = *src;
+ rand32 *= prime1;
+ rand32 ^= prime2;
+ rand32 = DiB_rotl32(rand32, 13);
+ *src = rand32;
+ return rand32 >> 5;
+}
+
+static void DiB_shuffle(const char** fileNamesTable, unsigned nbFiles) {
+ /* Initialize the pseudorandom number generator */
+ U32 seed = 0xFD2FB528;
+ unsigned i;
+ for (i = nbFiles - 1; i > 0; --i) {
+ unsigned const j = DiB_rand(&seed) % (i + 1);
+ const char* tmp = fileNamesTable[j];
+ fileNamesTable[j] = fileNamesTable[i];
+ fileNamesTable[i] = tmp;
+ }
+}
+
+
+/*-********************************************************
+* Dictionary training functions
+**********************************************************/
+static size_t DiB_findMaxMem(unsigned long long requiredMem)
+{
+ size_t const step = 8 MB;
+ void* testmem = NULL;
+
+ requiredMem = (((requiredMem >> 23) + 1) << 23);
+ requiredMem += step;
+ if (requiredMem > maxMemory) requiredMem = maxMemory;
+
+ while (!testmem) {
+ testmem = malloc((size_t)requiredMem);
+ requiredMem -= step;
+ }
+
+ free(testmem);
+ return (size_t)requiredMem;
+}
+
+
+static void DiB_fillNoise(void* buffer, size_t length)
+{
+ unsigned const prime1 = 2654435761U;
+ unsigned const prime2 = 2246822519U;
+ unsigned acc = prime1;
+ size_t p=0;;
+
+ for (p=0; p<length; p++) {
+ acc *= prime2;
+ ((unsigned char*)buffer)[p] = (unsigned char)(acc >> 21);
+ }
+}
+
+
+static void DiB_saveDict(const char* dictFileName,
+ const void* buff, size_t buffSize)
+{
+ FILE* const f = fopen(dictFileName, "wb");
+ if (f==NULL) EXM_THROW(3, "cannot open %s ", dictFileName);
+
+ { size_t const n = fwrite(buff, 1, buffSize, f);
+ if (n!=buffSize) EXM_THROW(4, "%s : write error", dictFileName) }
+
+ { size_t const n = (size_t)fclose(f);
+ if (n!=0) EXM_THROW(5, "%s : flush error", dictFileName) }
+}
+
+
+static int g_tooLargeSamples = 0;
+static U64 DiB_getTotalCappedFileSize(const char** fileNamesTable, unsigned nbFiles)
+{
+ U64 total = 0;
+ unsigned n;
+ for (n=0; n<nbFiles; n++) {
+ U64 const fileSize = UTIL_getFileSize(fileNamesTable[n]);
+ U64 const cappedFileSize = MIN(fileSize, SAMPLESIZE_MAX);
+ total += cappedFileSize;
+ g_tooLargeSamples |= (fileSize > 2*SAMPLESIZE_MAX);
+ }
+ return total;
+}
+
+
+/*! ZDICT_trainFromBuffer_unsafe() :
+ Strictly Internal use only !!
+ Same as ZDICT_trainFromBuffer_advanced(), but does not control `samplesBuffer`.
+ `samplesBuffer` must be followed by noisy guard band to avoid out-of-buffer reads.
+ @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
+ or an error code.
+*/
+size_t ZDICT_trainFromBuffer_unsafe(void* dictBuffer, size_t dictBufferCapacity,
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
+ ZDICT_params_t parameters);
+
+
+int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
+ const char** fileNamesTable, unsigned nbFiles,
+ ZDICT_params_t *params, COVER_params_t *coverParams,
+ int optimizeCover)
+{
+ void* const dictBuffer = malloc(maxDictSize);
+ size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
+ unsigned long long const totalSizeToLoad = DiB_getTotalCappedFileSize(fileNamesTable, nbFiles);
+ size_t const memMult = params ? MEMMULT : COVER_MEMMULT;
+ size_t const maxMem = DiB_findMaxMem(totalSizeToLoad * memMult) / memMult;
+ size_t benchedSize = (size_t) MIN ((unsigned long long)maxMem, totalSizeToLoad);
+ void* const srcBuffer = malloc(benchedSize+NOISELENGTH);
+ int result = 0;
+
+ /* Checks */
+ if (params) g_displayLevel = params->notificationLevel;
+ else if (coverParams) g_displayLevel = coverParams->notificationLevel;
+ else EXM_THROW(13, "Neither dictionary algorith selected"); /* should not happen */
+ if ((!fileSizes) || (!srcBuffer) || (!dictBuffer)) EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */
+ if (g_tooLargeSamples) {
+ DISPLAYLEVEL(2, "! Warning : some samples are very large \n");
+ DISPLAYLEVEL(2, "! Note that dictionary is only useful for small files or beginning of large files. \n");
+ DISPLAYLEVEL(2, "! As a consequence, only the first %u bytes of each file are loaded \n", SAMPLESIZE_MAX);
+ }
+ if ((nbFiles < 5) || (totalSizeToLoad < 9 * (unsigned long long)maxDictSize)) {
+ DISPLAYLEVEL(2, "! Warning : nb of samples too low for proper processing ! \n");
+ DISPLAYLEVEL(2, "! Please provide _one file per sample_. \n");
+ DISPLAYLEVEL(2, "! Do not concatenate samples together into a single file, \n");
+ DISPLAYLEVEL(2, "! as dictBuilder will be unable to find the beginning of each sample, \n");
+ DISPLAYLEVEL(2, "! resulting in poor dictionary quality. \n");
+ }
+
+ /* init */
+ if (benchedSize < totalSizeToLoad)
+ DISPLAYLEVEL(1, "Not enough memory; training on %u MB only...\n", (unsigned)(benchedSize >> 20));
+
+ /* Load input buffer */
+ DISPLAYLEVEL(3, "Shuffling input files\n");
+ DiB_shuffle(fileNamesTable, nbFiles);
+ nbFiles = DiB_loadFiles(srcBuffer, &benchedSize, fileSizes, fileNamesTable, nbFiles);
+
+ {
+ size_t dictSize;
+ if (params) {
+ DiB_fillNoise((char*)srcBuffer + benchedSize, NOISELENGTH); /* guard band, for end of buffer condition */
+ dictSize = ZDICT_trainFromBuffer_unsafe(dictBuffer, maxDictSize,
+ srcBuffer, fileSizes, nbFiles,
+ *params);
+ } else if (optimizeCover) {
+ dictSize = COVER_optimizeTrainFromBuffer(
+ dictBuffer, maxDictSize, srcBuffer, fileSizes, nbFiles,
+ coverParams);
+ if (!ZDICT_isError(dictSize)) {
+ DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\n", coverParams->k, coverParams->d, coverParams->steps);
+ }
+ } else {
+ dictSize = COVER_trainFromBuffer(dictBuffer, maxDictSize,
+ srcBuffer, fileSizes, nbFiles,
+ *coverParams);
+ }
+ if (ZDICT_isError(dictSize)) {
+ DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */
+ result = 1;
+ goto _cleanup;
+ }
+ /* save dict */
+ DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (U32)dictSize, dictFileName);
+ DiB_saveDict(dictFileName, dictBuffer, dictSize);
+ }
+
+ /* clean up */
+_cleanup:
+ free(srcBuffer);
+ free(dictBuffer);
+ free(fileSizes);
+ return result;
+}
diff --git a/programs/dibio.h b/programs/dibio.h
new file mode 100644
index 000000000000..e61d0042c850
--- /dev/null
+++ b/programs/dibio.h
@@ -0,0 +1,38 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+/* This library is designed for a single-threaded console application.
+* It exit() and printf() into stderr when it encounters an error condition. */
+
+#ifndef DIBIO_H_003
+#define DIBIO_H_003
+
+
+/*-*************************************
+* Dependencies
+***************************************/
+#define ZDICT_STATIC_LINKING_ONLY
+#include "zdict.h" /* ZDICT_params_t */
+
+
+/*-*************************************
+* Public functions
+***************************************/
+/*! DiB_trainFromFiles() :
+ Train a dictionary from a set of files provided by `fileNamesTable`.
+ Resulting dictionary is written into file `dictFileName`.
+ `parameters` is optional and can be provided with values set to 0, meaning "default".
+ @return : 0 == ok. Any other : error.
+*/
+int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
+ const char** fileNamesTable, unsigned nbFiles,
+ ZDICT_params_t *params, COVER_params_t *coverParams,
+ int optimizeCover);
+
+#endif
diff --git a/programs/fileio.c b/programs/fileio.c
new file mode 100644
index 000000000000..e6481f1fa726
--- /dev/null
+++ b/programs/fileio.c
@@ -0,0 +1,1200 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+
+/* *************************************
+* Compiler Options
+***************************************/
+#ifdef _MSC_VER /* Visual */
+# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
+# pragma warning(disable : 4204) /* non-constant aggregate initializer */
+#endif
+#if defined(__MINGW32__) && !defined(_POSIX_SOURCE)
+# define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */
+#endif
+
+
+/*-*************************************
+* Includes
+***************************************/
+#include "platform.h" /* Large Files support, SET_BINARY_MODE */
+#include "util.h" /* UTIL_getFileSize */
+#include <stdio.h> /* fprintf, fopen, fread, _fileno, stdin, stdout */
+#include <stdlib.h> /* malloc, free */
+#include <string.h> /* strcmp, strlen */
+#include <time.h> /* clock */
+#include <errno.h> /* errno */
+
+#include "mem.h"
+#include "fileio.h"
+#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
+#include "zstd.h"
+#ifdef ZSTD_MULTITHREAD
+# include "zstdmt_compress.h"
+#endif
+#if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS)
+# include <zlib.h>
+# if !defined(z_const)
+# define z_const
+# endif
+#endif
+#if defined(ZSTD_LZMACOMPRESS) || defined(ZSTD_LZMADECOMPRESS)
+# include <lzma.h>
+#endif
+
+
+/*-*************************************
+* Constants
+***************************************/
+#define KB *(1<<10)
+#define MB *(1<<20)
+#define GB *(1U<<30)
+
+#define _1BIT 0x01
+#define _2BITS 0x03
+#define _3BITS 0x07
+#define _4BITS 0x0F
+#define _6BITS 0x3F
+#define _8BITS 0xFF
+
+#define BLOCKSIZE (128 KB)
+#define ROLLBUFFERSIZE (BLOCKSIZE*8*64)
+
+#define FIO_FRAMEHEADERSIZE 5 /* as a define, because needed to allocated table on stack */
+#define FSE_CHECKSUM_SEED 0
+
+#define CACHELINE 64
+
+#define MAX_DICT_SIZE (8 MB) /* protection against large input (attack scenario) */
+
+#define FNSPACE 30
+
+
+/*-*************************************
+* Macros
+***************************************/
+#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
+static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
+void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
+
+#define DISPLAYUPDATE(l, ...) { if (g_displayLevel>=l) { \
+ if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \
+ { g_time = clock(); DISPLAY(__VA_ARGS__); \
+ if (g_displayLevel>=4) fflush(stdout); } } }
+static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
+static clock_t g_time = 0;
+
+#define MIN(a,b) ((a) < (b) ? (a) : (b))
+
+/* ************************************************************
+* Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW
+***************************************************************/
+#if defined(_MSC_VER) && _MSC_VER >= 1400
+# define LONG_SEEK _fseeki64
+#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */
+# define LONG_SEEK fseeko
+#elif defined(__MINGW32__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) && defined(__MSVCRT__)
+# define LONG_SEEK fseeko64
+#elif defined(_WIN32) && !defined(__DJGPP__)
+# include <windows.h>
+ static int LONG_SEEK(FILE* file, __int64 offset, int origin) {
+ LARGE_INTEGER off;
+ DWORD method;
+ off.QuadPart = offset;
+ if (origin == SEEK_END)
+ method = FILE_END;
+ else if (origin == SEEK_CUR)
+ method = FILE_CURRENT;
+ else
+ method = FILE_BEGIN;
+
+ if (SetFilePointerEx((HANDLE) _get_osfhandle(_fileno(file)), off, NULL, method))
+ return 0;
+ else
+ return -1;
+ }
+#else
+# define LONG_SEEK fseek
+#endif
+
+
+/*-*************************************
+* Local Parameters - Not thread safe
+***************************************/
+static FIO_compressionType_t g_compressionType = FIO_zstdCompression;
+void FIO_setCompressionType(FIO_compressionType_t compressionType) { g_compressionType = compressionType; }
+static U32 g_overwrite = 0;
+void FIO_overwriteMode(void) { g_overwrite=1; }
+static U32 g_sparseFileSupport = 1; /* 0 : no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
+void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; }
+static U32 g_dictIDFlag = 1;
+void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; }
+static U32 g_checksumFlag = 1;
+void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; }
+static U32 g_removeSrcFile = 0;
+void FIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); }
+static U32 g_memLimit = 0;
+void FIO_setMemLimit(unsigned memLimit) { g_memLimit = memLimit; }
+static U32 g_nbThreads = 1;
+void FIO_setNbThreads(unsigned nbThreads) {
+#ifndef ZSTD_MULTITHREAD
+ if (nbThreads > 1) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n");
+#endif
+ g_nbThreads = nbThreads;
+}
+static U32 g_blockSize = 0;
+void FIO_setBlockSize(unsigned blockSize) {
+ if (blockSize && g_nbThreads==1)
+ DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n");
+#ifdef ZSTD_MULTITHREAD
+ if (blockSize-1 < ZSTDMT_SECTION_SIZE_MIN-1) /* intentional underflow */
+ DISPLAYLEVEL(2, "Note : minimum block size is %u KB \n", (ZSTDMT_SECTION_SIZE_MIN>>10));
+#endif
+ g_blockSize = blockSize;
+}
+#define FIO_OVERLAP_LOG_NOTSET 9999
+static U32 g_overlapLog = FIO_OVERLAP_LOG_NOTSET;
+void FIO_setOverlapLog(unsigned overlapLog){
+ if (overlapLog && g_nbThreads==1)
+ DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n");
+ g_overlapLog = overlapLog;
+}
+
+
+/*-*************************************
+* Exceptions
+***************************************/
+#ifndef DEBUG
+# define DEBUG 0
+#endif
+#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
+#define EXM_THROW(error, ...) \
+{ \
+ DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
+ DISPLAYLEVEL(1, "Error %i : ", error); \
+ DISPLAYLEVEL(1, __VA_ARGS__); \
+ DISPLAYLEVEL(1, " \n"); \
+ exit(error); \
+}
+
+
+/*-*************************************
+* Functions
+***************************************/
+/** FIO_openSrcFile() :
+ * condition : `dstFileName` must be non-NULL.
+ * @result : FILE* to `dstFileName`, or NULL if it fails */
+static FILE* FIO_openSrcFile(const char* srcFileName)
+{
+ FILE* f;
+
+ if (!strcmp (srcFileName, stdinmark)) {
+ DISPLAYLEVEL(4,"Using stdin for input\n");
+ f = stdin;
+ SET_BINARY_MODE(stdin);
+ } else {
+ if (!UTIL_isRegFile(srcFileName)) {
+ DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", srcFileName);
+ return NULL;
+ }
+ f = fopen(srcFileName, "rb");
+ if ( f==NULL ) DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
+ }
+
+ return f;
+}
+
+/** FIO_openDstFile() :
+ * condition : `dstFileName` must be non-NULL.
+ * @result : FILE* to `dstFileName`, or NULL if it fails */
+static FILE* FIO_openDstFile(const char* dstFileName)
+{
+ FILE* f;
+
+ if (!strcmp (dstFileName, stdoutmark)) {
+ DISPLAYLEVEL(4,"Using stdout for output\n");
+ f = stdout;
+ SET_BINARY_MODE(stdout);
+ if (g_sparseFileSupport==1) {
+ g_sparseFileSupport = 0;
+ DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n");
+ }
+ } else {
+ if (!g_overwrite && strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */
+ f = fopen( dstFileName, "rb" );
+ if (f != 0) { /* dest file exists, prompt for overwrite authorization */
+ fclose(f);
+ if (g_displayLevel <= 1) {
+ /* No interaction possible */
+ DISPLAY("zstd: %s already exists; not overwritten \n", dstFileName);
+ return NULL;
+ }
+ DISPLAY("zstd: %s already exists; do you wish to overwrite (y/N) ? ", dstFileName);
+ { int ch = getchar();
+ if ((ch!='Y') && (ch!='y')) {
+ DISPLAY(" not overwritten \n");
+ return NULL;
+ }
+ while ((ch!=EOF) && (ch!='\n')) ch = getchar(); /* flush rest of input line */
+ } } }
+ f = fopen( dstFileName, "wb" );
+ if (f==NULL) DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno));
+ }
+
+ return f;
+}
+
+
+/*! FIO_loadFile() :
+* creates a buffer, pointed by `*bufferPtr`,
+* loads `filename` content into it,
+* up to MAX_DICT_SIZE bytes.
+* @return : loaded size
+*/
+static size_t FIO_loadFile(void** bufferPtr, const char* fileName)
+{
+ FILE* fileHandle;
+ U64 fileSize;
+
+ *bufferPtr = NULL;
+ if (fileName == NULL) return 0;
+
+ DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
+ fileHandle = fopen(fileName, "rb");
+ if (fileHandle==0) EXM_THROW(31, "zstd: %s: %s", fileName, strerror(errno));
+ fileSize = UTIL_getFileSize(fileName);
+ if (fileSize > MAX_DICT_SIZE) {
+ int seekResult;
+ if (fileSize > 1 GB) EXM_THROW(32, "Dictionary file %s is too large", fileName); /* avoid extreme cases */
+ DISPLAYLEVEL(2,"Dictionary %s is too large : using last %u bytes only \n", fileName, (U32)MAX_DICT_SIZE);
+ seekResult = fseek(fileHandle, (long int)(fileSize-MAX_DICT_SIZE), SEEK_SET); /* use end of file */
+ if (seekResult != 0) EXM_THROW(33, "zstd: %s: %s", fileName, strerror(errno));
+ fileSize = MAX_DICT_SIZE;
+ }
+ *bufferPtr = malloc((size_t)fileSize);
+ if (*bufferPtr==NULL) EXM_THROW(34, "zstd: %s", strerror(errno));
+ { size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
+ if (readSize!=fileSize) EXM_THROW(35, "Error reading dictionary file %s", fileName); }
+ fclose(fileHandle);
+ return (size_t)fileSize;
+}
+
+#ifndef ZSTD_NOCOMPRESS
+
+/*-**********************************************************************
+* Compression
+************************************************************************/
+typedef struct {
+ FILE* srcFile;
+ FILE* dstFile;
+ void* srcBuffer;
+ size_t srcBufferSize;
+ void* dstBuffer;
+ size_t dstBufferSize;
+#ifdef ZSTD_MULTITHREAD
+ ZSTDMT_CCtx* cctx;
+#else
+ ZSTD_CStream* cctx;
+#endif
+} cRess_t;
+
+static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
+ U64 srcSize, int srcRegFile,
+ ZSTD_compressionParameters* comprParams) {
+ cRess_t ress;
+ memset(&ress, 0, sizeof(ress));
+
+#ifdef ZSTD_MULTITHREAD
+ ress.cctx = ZSTDMT_createCCtx(g_nbThreads);
+ if (ress.cctx == NULL) EXM_THROW(30, "zstd: allocation error : can't create ZSTD_CStream");
+ if ((cLevel==ZSTD_maxCLevel()) && (g_overlapLog==FIO_OVERLAP_LOG_NOTSET))
+ ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, 9); /* use complete window for overlap */
+ if (g_overlapLog != FIO_OVERLAP_LOG_NOTSET)
+ ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, g_overlapLog);
+#else
+ ress.cctx = ZSTD_createCStream();
+ if (ress.cctx == NULL) EXM_THROW(30, "zstd: allocation error : can't create ZSTD_CStream");
+#endif
+ ress.srcBufferSize = ZSTD_CStreamInSize();
+ ress.srcBuffer = malloc(ress.srcBufferSize);
+ ress.dstBufferSize = ZSTD_CStreamOutSize();
+ ress.dstBuffer = malloc(ress.dstBufferSize);
+ if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "zstd: allocation error : not enough memory");
+
+ /* dictionary */
+ { void* dictBuffer;
+ size_t const dictBuffSize = FIO_loadFile(&dictBuffer, dictFileName);
+ if (dictFileName && (dictBuffer==NULL)) EXM_THROW(32, "zstd: allocation error : can't create dictBuffer");
+ { ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
+ params.fParams.contentSizeFlag = srcRegFile;
+ params.fParams.checksumFlag = g_checksumFlag;
+ params.fParams.noDictIDFlag = !g_dictIDFlag;
+ if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog;
+ if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog;
+ if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog;
+ if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog;
+ if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength;
+ if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength;
+ if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy)(comprParams->strategy - 1);
+#ifdef ZSTD_MULTITHREAD
+ { size_t const errorCode = ZSTDMT_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize);
+ if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode));
+ ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_sectionSize, g_blockSize);
+#else
+ { size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize);
+ if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode));
+#endif
+ } }
+ free(dictBuffer);
+ }
+
+ return ress;
+}
+
+static void FIO_freeCResources(cRess_t ress)
+{
+ free(ress.srcBuffer);
+ free(ress.dstBuffer);
+#ifdef ZSTD_MULTITHREAD
+ ZSTDMT_freeCCtx(ress.cctx);
+#else
+ ZSTD_freeCStream(ress.cctx); /* never fails */
+#endif
+}
+
+
+#ifdef ZSTD_GZCOMPRESS
+static unsigned long long FIO_compressGzFrame(cRess_t* ress, const char* srcFileName, U64 const srcFileSize, int compressionLevel, U64* readsize)
+{
+ unsigned long long inFileSize = 0, outFileSize = 0;
+ z_stream strm;
+ int ret;
+
+ if (compressionLevel > Z_BEST_COMPRESSION) compressionLevel = Z_BEST_COMPRESSION;
+
+ strm.zalloc = Z_NULL;
+ strm.zfree = Z_NULL;
+ strm.opaque = Z_NULL;
+
+ ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED, 15 /* maxWindowLogSize */ + 16 /* gzip only */, 8, Z_DEFAULT_STRATEGY); /* see http://www.zlib.net/manual.html */
+ if (ret != Z_OK) EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret);
+
+ strm.next_in = 0;
+ strm.avail_in = 0;
+ strm.next_out = (Bytef*)ress->dstBuffer;
+ strm.avail_out = (uInt)ress->dstBufferSize;
+
+ while (1) {
+ if (strm.avail_in == 0) {
+ size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile);
+ if (inSize == 0) break;
+ inFileSize += inSize;
+ strm.next_in = (z_const unsigned char*)ress->srcBuffer;
+ strm.avail_in = (uInt)inSize;
+ }
+ ret = deflate(&strm, Z_NO_FLUSH);
+ if (ret != Z_OK) EXM_THROW(72, "zstd: %s: deflate error %d \n", srcFileName, ret);
+ { size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
+ if (decompBytes) {
+ if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(73, "Write error : cannot write to output file");
+ outFileSize += decompBytes;
+ strm.next_out = (Bytef*)ress->dstBuffer;
+ strm.avail_out = (uInt)ress->dstBufferSize;
+ }
+ }
+ if (!srcFileSize) DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", (U32)(inFileSize>>20), (double)outFileSize/inFileSize*100)
+ else DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", (U32)(inFileSize>>20), (U32)(srcFileSize>>20), (double)outFileSize/inFileSize*100);
+ }
+
+ while (1) {
+ ret = deflate(&strm, Z_FINISH);
+ { size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
+ if (decompBytes) {
+ if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(75, "Write error : cannot write to output file");
+ outFileSize += decompBytes;
+ strm.next_out = (Bytef*)ress->dstBuffer;
+ strm.avail_out = (uInt)ress->dstBufferSize;
+ }
+ }
+ if (ret == Z_STREAM_END) break;
+ if (ret != Z_BUF_ERROR) EXM_THROW(77, "zstd: %s: deflate error %d \n", srcFileName, ret);
+ }
+
+ ret = deflateEnd(&strm);
+ if (ret != Z_OK) EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret);
+ *readsize = inFileSize;
+
+ return outFileSize;
+}
+#endif
+
+
+#ifdef ZSTD_LZMACOMPRESS
+static unsigned long long FIO_compressLzmaFrame(cRess_t* ress, const char* srcFileName, U64 const srcFileSize, int compressionLevel, U64* readsize, int plain_lzma)
+{
+ unsigned long long inFileSize = 0, outFileSize = 0;
+ lzma_stream strm = LZMA_STREAM_INIT;
+ lzma_action action = LZMA_RUN;
+ lzma_ret ret;
+
+ if (compressionLevel < 0) compressionLevel = 0;
+ if (compressionLevel > 9) compressionLevel = 9;
+
+ if (plain_lzma) {
+ lzma_options_lzma opt_lzma;
+ if (lzma_lzma_preset(&opt_lzma, compressionLevel)) EXM_THROW(71, "zstd: %s: lzma_lzma_preset error", srcFileName);
+ ret = lzma_alone_encoder(&strm, &opt_lzma); /* LZMA */
+ if (ret != LZMA_OK) EXM_THROW(71, "zstd: %s: lzma_alone_encoder error %d", srcFileName, ret);
+ } else {
+ ret = lzma_easy_encoder(&strm, compressionLevel, LZMA_CHECK_CRC64); /* XZ */
+ if (ret != LZMA_OK) EXM_THROW(71, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret);
+ }
+
+ strm.next_in = 0;
+ strm.avail_in = 0;
+ strm.next_out = ress->dstBuffer;
+ strm.avail_out = ress->dstBufferSize;
+
+ while (1) {
+ if (strm.avail_in == 0) {
+ size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile);
+ if (inSize == 0) action = LZMA_FINISH;
+ inFileSize += inSize;
+ strm.next_in = ress->srcBuffer;
+ strm.avail_in = inSize;
+ }
+
+ ret = lzma_code(&strm, action);
+
+ if (ret != LZMA_OK && ret != LZMA_STREAM_END) EXM_THROW(72, "zstd: %s: lzma_code encoding error %d", srcFileName, ret);
+ { size_t const compBytes = ress->dstBufferSize - strm.avail_out;
+ if (compBytes) {
+ if (fwrite(ress->dstBuffer, 1, compBytes, ress->dstFile) != compBytes) EXM_THROW(73, "Write error : cannot write to output file");
+ outFileSize += compBytes;
+ strm.next_out = ress->dstBuffer;
+ strm.avail_out = ress->dstBufferSize;
+ }
+ }
+ if (!srcFileSize) DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", (U32)(inFileSize>>20), (double)outFileSize/inFileSize*100)
+ else DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", (U32)(inFileSize>>20), (U32)(srcFileSize>>20), (double)outFileSize/inFileSize*100);
+ if (ret == LZMA_STREAM_END) break;
+ }
+
+ lzma_end(&strm);
+ *readsize = inFileSize;
+
+ return outFileSize;
+}
+#endif
+
+
+/*! FIO_compressFilename_internal() :
+ * same as FIO_compressFilename_extRess(), with `ress.desFile` already opened.
+ * @return : 0 : compression completed correctly,
+ * 1 : missing or pb opening srcFileName
+ */
+static int FIO_compressFilename_internal(cRess_t ress,
+ const char* dstFileName, const char* srcFileName, int compressionLevel)
+{
+ FILE* const srcFile = ress.srcFile;
+ FILE* const dstFile = ress.dstFile;
+ U64 readsize = 0;
+ U64 compressedfilesize = 0;
+ U64 const fileSize = UTIL_getFileSize(srcFileName);
+
+ switch (g_compressionType) {
+ case FIO_zstdCompression:
+ break;
+ case FIO_gzipCompression:
+#ifdef ZSTD_GZCOMPRESS
+ compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize);
+#else
+ (void)compressionLevel;
+ EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n", srcFileName);
+#endif
+ goto finish;
+ case FIO_xzCompression:
+ case FIO_lzmaCompression:
+#ifdef ZSTD_LZMACOMPRESS
+ compressedfilesize = FIO_compressLzmaFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize, g_compressionType==FIO_lzmaCompression);
+#else
+ (void)compressionLevel;
+ EXM_THROW(20, "zstd: %s: file cannot be compressed as xz/lzma (zstd compiled without ZSTD_LZMACOMPRESS) -- ignored \n", srcFileName);
+#endif
+ goto finish;
+ }
+
+ /* init */
+#ifdef ZSTD_MULTITHREAD
+ { size_t const resetError = ZSTDMT_resetCStream(ress.cctx, fileSize);
+#else
+ { size_t const resetError = ZSTD_resetCStream(ress.cctx, fileSize);
+#endif
+ if (ZSTD_isError(resetError)) EXM_THROW(21, "Error initializing compression : %s", ZSTD_getErrorName(resetError));
+ }
+
+ /* Main compression loop */
+ while (1) {
+ /* Fill input Buffer */
+ size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile);
+ if (inSize==0) break;
+ readsize += inSize;
+
+ { ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 };
+ while (inBuff.pos != inBuff.size) { /* note : is there any possibility of endless loop ? for example, if outBuff is not large enough ? */
+ ZSTD_outBuffer outBuff= { ress.dstBuffer, ress.dstBufferSize, 0 };
+#ifdef ZSTD_MULTITHREAD
+ size_t const result = ZSTDMT_compressStream(ress.cctx, &outBuff, &inBuff);
+#else
+ size_t const result = ZSTD_compressStream(ress.cctx, &outBuff, &inBuff);
+#endif
+ if (ZSTD_isError(result)) EXM_THROW(23, "Compression error : %s ", ZSTD_getErrorName(result));
+
+ /* Write compressed stream */
+ if (outBuff.pos) {
+ size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
+ if (sizeCheck!=outBuff.pos) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName);
+ compressedfilesize += outBuff.pos;
+ } } }
+#ifdef ZSTD_MULTITHREAD
+ if (!fileSize) DISPLAYUPDATE(2, "\rRead : %u MB", (U32)(readsize>>20))
+ else DISPLAYUPDATE(2, "\rRead : %u / %u MB", (U32)(readsize>>20), (U32)(fileSize>>20));
+#else
+ if (!fileSize) DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", (U32)(readsize>>20), (double)compressedfilesize/readsize*100)
+ else DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", (U32)(readsize>>20), (U32)(fileSize>>20), (double)compressedfilesize/readsize*100);
+#endif
+ }
+
+ /* End of Frame */
+ { size_t result = 1;
+ while (result!=0) { /* note : is there any possibility of endless loop ? */
+ ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
+#ifdef ZSTD_MULTITHREAD
+ result = ZSTDMT_endStream(ress.cctx, &outBuff);
+#else
+ result = ZSTD_endStream(ress.cctx, &outBuff);
+#endif
+ if (ZSTD_isError(result)) EXM_THROW(26, "Compression error during frame end : %s", ZSTD_getErrorName(result));
+ { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
+ if (sizeCheck!=outBuff.pos) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); }
+ compressedfilesize += outBuff.pos;
+ }
+ }
+
+finish:
+ /* Status */
+ DISPLAYLEVEL(2, "\r%79s\r", "");
+ DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", srcFileName,
+ (double)compressedfilesize/(readsize+(!readsize) /* avoid div by zero */ )*100,
+ (unsigned long long)readsize, (unsigned long long) compressedfilesize,
+ dstFileName);
+
+ return 0;
+}
+
+
+/*! FIO_compressFilename_srcFile() :
+ * note : ress.destFile already opened
+ * @return : 0 : compression completed correctly,
+ * 1 : missing or pb opening srcFileName
+ */
+static int FIO_compressFilename_srcFile(cRess_t ress,
+ const char* dstFileName, const char* srcFileName, int compressionLevel)
+{
+ int result;
+
+ /* File check */
+ if (UTIL_isDirectory(srcFileName)) {
+ DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName);
+ return 1;
+ }
+
+ ress.srcFile = FIO_openSrcFile(srcFileName);
+ if (!ress.srcFile) return 1; /* srcFile could not be opened */
+
+ result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, compressionLevel);
+
+ fclose(ress.srcFile);
+ if (g_removeSrcFile /* --rm */ && !result && strcmp(srcFileName, stdinmark)) {
+ if (remove(srcFileName))
+ EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno));
+ }
+ return result;
+}
+
+
+/*! FIO_compressFilename_dstFile() :
+ * @return : 0 : compression completed correctly,
+ * 1 : pb
+ */
+static int FIO_compressFilename_dstFile(cRess_t ress,
+ const char* dstFileName, const char* srcFileName, int compressionLevel)
+{
+ int result;
+ stat_t statbuf;
+ int stat_result = 0;
+
+ ress.dstFile = FIO_openDstFile(dstFileName);
+ if (ress.dstFile==NULL) return 1; /* could not open dstFileName */
+
+ if (strcmp (srcFileName, stdinmark) && UTIL_getFileStat(srcFileName, &statbuf)) stat_result = 1;
+ result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, compressionLevel);
+
+ if (fclose(ress.dstFile)) { DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result=1; } /* error closing dstFile */
+ if (result!=0) { if (remove(dstFileName)) EXM_THROW(1, "zstd: %s: %s", dstFileName, strerror(errno)); } /* remove operation artefact */
+ else if (strcmp (dstFileName, stdoutmark) && stat_result) UTIL_setFileStat(dstFileName, &statbuf);
+ return result;
+}
+
+
+int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
+ const char* dictFileName, int compressionLevel, ZSTD_compressionParameters* comprParams)
+{
+ clock_t const start = clock();
+ U64 const srcSize = UTIL_getFileSize(srcFileName);
+ int const regFile = UTIL_isRegFile(srcFileName);
+
+ cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, regFile, comprParams);
+ int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel);
+
+ double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC;
+ DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds);
+
+ FIO_freeCResources(ress);
+ return result;
+}
+
+
+int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles,
+ const char* suffix,
+ const char* dictFileName, int compressionLevel,
+ ZSTD_compressionParameters* comprParams)
+{
+ int missed_files = 0;
+ size_t dfnSize = FNSPACE;
+ char* dstFileName = (char*)malloc(FNSPACE);
+ size_t const suffixSize = suffix ? strlen(suffix) : 0;
+ U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ;
+ int const regFile = (nbFiles != 1) ? 0 : UTIL_isRegFile(inFileNamesTable[0]);
+ cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, regFile, comprParams);
+
+ /* init */
+ if (dstFileName==NULL) EXM_THROW(27, "FIO_compressMultipleFilenames : allocation error for dstFileName");
+ if (suffix == NULL) EXM_THROW(28, "FIO_compressMultipleFilenames : dst unknown"); /* should never happen */
+
+ /* loop on each file */
+ if (!strcmp(suffix, stdoutmark)) {
+ unsigned u;
+ ress.dstFile = stdout;
+ SET_BINARY_MODE(stdout);
+ for (u=0; u<nbFiles; u++)
+ missed_files += FIO_compressFilename_srcFile(ress, stdoutmark, inFileNamesTable[u], compressionLevel);
+ if (fclose(ress.dstFile)) EXM_THROW(29, "Write error : cannot properly close stdout");
+ } else {
+ unsigned u;
+ for (u=0; u<nbFiles; u++) {
+ size_t ifnSize = strlen(inFileNamesTable[u]);
+ if (dfnSize <= ifnSize+suffixSize+1) { free(dstFileName); dfnSize = ifnSize + 20; dstFileName = (char*)malloc(dfnSize); }
+ strcpy(dstFileName, inFileNamesTable[u]);
+ strcat(dstFileName, suffix);
+ missed_files += FIO_compressFilename_dstFile(ress, dstFileName, inFileNamesTable[u], compressionLevel);
+ } }
+
+ /* Close & Free */
+ FIO_freeCResources(ress);
+ free(dstFileName);
+
+ return missed_files;
+}
+
+#endif /* #ifndef ZSTD_NOCOMPRESS */
+
+
+
+#ifndef ZSTD_NODECOMPRESS
+
+/* **************************************************************************
+* Decompression
+****************************************************************************/
+typedef struct {
+ void* srcBuffer;
+ size_t srcBufferLoaded;
+ size_t srcBufferSize;
+ void* dstBuffer;
+ size_t dstBufferSize;
+ ZSTD_DStream* dctx;
+ FILE* dstFile;
+} dRess_t;
+
+static dRess_t FIO_createDResources(const char* dictFileName)
+{
+ dRess_t ress;
+ memset(&ress, 0, sizeof(ress));
+
+ /* Allocation */
+ ress.dctx = ZSTD_createDStream();
+ if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZSTD_DStream");
+ ZSTD_setDStreamParameter(ress.dctx, DStream_p_maxWindowSize, g_memLimit);
+ ress.srcBufferSize = ZSTD_DStreamInSize();
+ ress.srcBuffer = malloc(ress.srcBufferSize);
+ ress.dstBufferSize = ZSTD_DStreamOutSize();
+ ress.dstBuffer = malloc(ress.dstBufferSize);
+ if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
+
+ /* dictionary */
+ { void* dictBuffer;
+ size_t const dictBufferSize = FIO_loadFile(&dictBuffer, dictFileName);
+ size_t const initError = ZSTD_initDStream_usingDict(ress.dctx, dictBuffer, dictBufferSize);
+ if (ZSTD_isError(initError)) EXM_THROW(61, "ZSTD_initDStream_usingDict error : %s", ZSTD_getErrorName(initError));
+ free(dictBuffer);
+ }
+
+ return ress;
+}
+
+static void FIO_freeDResources(dRess_t ress)
+{
+ size_t const errorCode = ZSTD_freeDStream(ress.dctx);
+ if (ZSTD_isError(errorCode)) EXM_THROW(69, "Error : can't free ZSTD_DStream context resource : %s", ZSTD_getErrorName(errorCode));
+ free(ress.srcBuffer);
+ free(ress.dstBuffer);
+}
+
+
+/** FIO_fwriteSparse() :
+* @return : storedSkips, to be provided to next call to FIO_fwriteSparse() of LZ4IO_fwriteSparseEnd() */
+static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips)
+{
+ const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */
+ size_t bufferSizeT = bufferSize / sizeof(size_t);
+ const size_t* const bufferTEnd = bufferT + bufferSizeT;
+ const size_t* ptrT = bufferT;
+ static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* 0-test re-attempted every 32 KB */
+
+ if (!g_sparseFileSupport) { /* normal write */
+ size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file);
+ if (sizeCheck != bufferSize) EXM_THROW(70, "Write error : cannot write decoded block");
+ return 0;
+ }
+
+ /* avoid int overflow */
+ if (storedSkips > 1 GB) {
+ int const seekResult = LONG_SEEK(file, 1 GB, SEEK_CUR);
+ if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)");
+ storedSkips -= 1 GB;
+ }
+
+ while (ptrT < bufferTEnd) {
+ size_t seg0SizeT = segmentSizeT;
+ size_t nb0T;
+
+ /* count leading zeros */
+ if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT;
+ bufferSizeT -= seg0SizeT;
+ for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ;
+ storedSkips += (unsigned)(nb0T * sizeof(size_t));
+
+ if (nb0T != seg0SizeT) { /* not all 0s */
+ int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
+ if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse");
+ storedSkips = 0;
+ seg0SizeT -= nb0T;
+ ptrT += nb0T;
+ { size_t const sizeCheck = fwrite(ptrT, sizeof(size_t), seg0SizeT, file);
+ if (sizeCheck != seg0SizeT) EXM_THROW(73, "Write error : cannot write decoded block");
+ } }
+ ptrT += seg0SizeT;
+ }
+
+ { static size_t const maskT = sizeof(size_t)-1;
+ if (bufferSize & maskT) { /* size not multiple of sizeof(size_t) : implies end of block */
+ const char* const restStart = (const char*)bufferTEnd;
+ const char* restPtr = restStart;
+ size_t restSize = bufferSize & maskT;
+ const char* const restEnd = restStart + restSize;
+ for ( ; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ;
+ storedSkips += (unsigned) (restPtr - restStart);
+ if (restPtr != restEnd) {
+ int seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
+ if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse");
+ storedSkips = 0;
+ { size_t const sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file);
+ if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(75, "Write error : cannot write decoded end of block");
+ } } } }
+
+ return storedSkips;
+}
+
+static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
+{
+ if (storedSkips-->0) { /* implies g_sparseFileSupport>0 */
+ int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
+ if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)");
+ { const char lastZeroByte[1] = { 0 };
+ size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file);
+ if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero");
+ } }
+}
+
+
+/** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode
+ @return : 0 (no error) */
+static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_t bufferSize, size_t alreadyLoaded)
+{
+ size_t const blockSize = MIN(64 KB, bufferSize);
+ size_t readFromInput = 1;
+ unsigned storedSkips = 0;
+
+ /* assumption : ress->srcBufferLoaded bytes already loaded and stored within buffer */
+ { size_t const sizeCheck = fwrite(buffer, 1, alreadyLoaded, foutput);
+ if (sizeCheck != alreadyLoaded) EXM_THROW(50, "Pass-through write error"); }
+
+ while (readFromInput) {
+ readFromInput = fread(buffer, 1, blockSize, finput);
+ storedSkips = FIO_fwriteSparse(foutput, buffer, readFromInput, storedSkips);
+ }
+
+ FIO_fwriteSparseEnd(foutput, storedSkips);
+ return 0;
+}
+
+
+/** FIO_decompressFrame() :
+ @return : size of decoded frame
+*/
+unsigned long long FIO_decompressFrame(dRess_t* ress,
+ FILE* finput,
+ U64 alreadyDecoded)
+{
+ U64 frameSize = 0;
+ U32 storedSkips = 0;
+
+ ZSTD_resetDStream(ress->dctx);
+
+ /* Header loading (optional, saves one loop) */
+ { size_t const toRead = 9;
+ if (ress->srcBufferLoaded < toRead)
+ ress->srcBufferLoaded += fread(((char*)ress->srcBuffer) + ress->srcBufferLoaded, 1, toRead - ress->srcBufferLoaded, finput);
+ }
+
+ /* Main decompression Loop */
+ while (1) {
+ ZSTD_inBuffer inBuff = { ress->srcBuffer, ress->srcBufferLoaded, 0 };
+ ZSTD_outBuffer outBuff= { ress->dstBuffer, ress->dstBufferSize, 0 };
+ size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff);
+ if (ZSTD_isError(readSizeHint)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(readSizeHint));
+
+ /* Write block */
+ storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, storedSkips);
+ frameSize += outBuff.pos;
+ DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)((alreadyDecoded+frameSize)>>20) );
+
+ if (inBuff.pos > 0) {
+ memmove(ress->srcBuffer, (char*)ress->srcBuffer + inBuff.pos, inBuff.size - inBuff.pos);
+ ress->srcBufferLoaded -= inBuff.pos;
+ }
+
+ if (readSizeHint == 0) break; /* end of frame */
+ if (inBuff.size != inBuff.pos) EXM_THROW(37, "Decoding error : should consume entire input");
+
+ /* Fill input buffer */
+ { size_t const toRead = MIN(readSizeHint, ress->srcBufferSize); /* support large skippable frames */
+ if (ress->srcBufferLoaded < toRead)
+ ress->srcBufferLoaded += fread(((char*)ress->srcBuffer) + ress->srcBufferLoaded, 1, toRead - ress->srcBufferLoaded, finput);
+ if (ress->srcBufferLoaded < toRead) EXM_THROW(39, "Read error : premature end");
+ } }
+
+ FIO_fwriteSparseEnd(ress->dstFile, storedSkips);
+
+ return frameSize;
+}
+
+
+#ifdef ZSTD_GZDECOMPRESS
+static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName)
+{
+ unsigned long long outFileSize = 0;
+ z_stream strm;
+ int flush = Z_NO_FLUSH;
+ int ret;
+
+ strm.zalloc = Z_NULL;
+ strm.zfree = Z_NULL;
+ strm.opaque = Z_NULL;
+ strm.next_in = 0;
+ strm.avail_in = 0;
+ if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK) return 0; /* see http://www.zlib.net/manual.html */
+
+ strm.next_out = (Bytef*)ress->dstBuffer;
+ strm.avail_out = (uInt)ress->dstBufferSize;
+ strm.avail_in = (uInt)ress->srcBufferLoaded;
+ strm.next_in = (z_const unsigned char*)ress->srcBuffer;
+
+ for ( ; ; ) {
+ if (strm.avail_in == 0) {
+ ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile);
+ if (ress->srcBufferLoaded == 0) flush = Z_FINISH;
+ strm.next_in = (z_const unsigned char*)ress->srcBuffer;
+ strm.avail_in = (uInt)ress->srcBufferLoaded;
+ }
+ ret = inflate(&strm, flush);
+ if (ret == Z_BUF_ERROR) EXM_THROW(39, "zstd: %s: premature end", srcFileName);
+ if (ret != Z_OK && ret != Z_STREAM_END) { DISPLAY("zstd: %s: inflate error %d \n", srcFileName, ret); return 0; }
+ { size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
+ if (decompBytes) {
+ if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(31, "Write error : cannot write to output file");
+ outFileSize += decompBytes;
+ strm.next_out = (Bytef*)ress->dstBuffer;
+ strm.avail_out = (uInt)ress->dstBufferSize;
+ }
+ }
+ if (ret == Z_STREAM_END) break;
+ }
+
+ if (strm.avail_in > 0) memmove(ress->srcBuffer, strm.next_in, strm.avail_in);
+ ress->srcBufferLoaded = strm.avail_in;
+ ret = inflateEnd(&strm);
+ if (ret != Z_OK) EXM_THROW(32, "zstd: %s: inflateEnd error %d", srcFileName, ret);
+ return outFileSize;
+}
+#endif
+
+
+#ifdef ZSTD_LZMADECOMPRESS
+static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName, int plain_lzma)
+{
+ unsigned long long outFileSize = 0;
+ lzma_stream strm = LZMA_STREAM_INIT;
+ lzma_action action = LZMA_RUN;
+ lzma_ret ret;
+
+ strm.next_in = 0;
+ strm.avail_in = 0;
+ if (plain_lzma) {
+ ret = lzma_alone_decoder(&strm, UINT64_MAX); /* LZMA */
+ } else {
+ ret = lzma_stream_decoder(&strm, UINT64_MAX, 0); /* XZ */
+ }
+
+ if (ret != LZMA_OK) EXM_THROW(71, "zstd: %s: lzma_alone_decoder/lzma_stream_decoder error %d", srcFileName, ret);
+
+ strm.next_out = ress->dstBuffer;
+ strm.avail_out = ress->dstBufferSize;
+ strm.avail_in = ress->srcBufferLoaded;
+ strm.next_in = ress->srcBuffer;
+
+ for ( ; ; ) {
+ if (strm.avail_in == 0) {
+ ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile);
+ if (ress->srcBufferLoaded == 0) action = LZMA_FINISH;
+ strm.next_in = ress->srcBuffer;
+ strm.avail_in = ress->srcBufferLoaded;
+ }
+ ret = lzma_code(&strm, action);
+
+ if (ret == LZMA_BUF_ERROR) EXM_THROW(39, "zstd: %s: premature end", srcFileName);
+ if (ret != LZMA_OK && ret != LZMA_STREAM_END) { DISPLAY("zstd: %s: lzma_code decoding error %d \n", srcFileName, ret); return 0; }
+ { size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
+ if (decompBytes) {
+ if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(31, "Write error : cannot write to output file");
+ outFileSize += decompBytes;
+ strm.next_out = ress->dstBuffer;
+ strm.avail_out = ress->dstBufferSize;
+ }
+ }
+ if (ret == LZMA_STREAM_END) break;
+ }
+
+ if (strm.avail_in > 0) memmove(ress->srcBuffer, strm.next_in, strm.avail_in);
+ ress->srcBufferLoaded = strm.avail_in;
+ lzma_end(&strm);
+ return outFileSize;
+}
+#endif
+
+
+/** FIO_decompressSrcFile() :
+ Decompression `srcFileName` into `ress.dstFile`
+ @return : 0 : OK
+ 1 : operation not started
+*/
+static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const char* srcFileName)
+{
+ FILE* srcFile;
+ unsigned readSomething = 0;
+ unsigned long long filesize = 0;
+
+ if (UTIL_isDirectory(srcFileName)) {
+ DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName);
+ return 1;
+ }
+
+ srcFile = FIO_openSrcFile(srcFileName);
+ if (srcFile==NULL) return 1;
+
+ /* for each frame */
+ for ( ; ; ) {
+ /* check magic number -> version */
+ size_t const toRead = 4;
+ const BYTE* buf = (const BYTE*)ress.srcBuffer;
+ if (ress.srcBufferLoaded < toRead)
+ ress.srcBufferLoaded += fread((char*)ress.srcBuffer + ress.srcBufferLoaded, (size_t)1, toRead - ress.srcBufferLoaded, srcFile);
+ if (ress.srcBufferLoaded==0) {
+ if (readSomething==0) { DISPLAY("zstd: %s: unexpected end of file \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */
+ break; /* no more input */
+ }
+ readSomething = 1; /* there is at least >= 4 bytes in srcFile */
+ if (ress.srcBufferLoaded < toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */
+ if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */
+#ifdef ZSTD_GZDECOMPRESS
+ unsigned long long const result = FIO_decompressGzFrame(&ress, srcFile, srcFileName);
+ if (result == 0) return 1;
+ filesize += result;
+#else
+ DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without ZSTD_GZDECOMPRESS) -- ignored \n", srcFileName);
+ return 1;
+#endif
+ } else if ((buf[0] == 0xFD && buf[1] == 0x37) /* xz magic number */
+ || (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */
+#ifdef ZSTD_LZMADECOMPRESS
+ unsigned long long const result = FIO_decompressLzmaFrame(&ress, srcFile, srcFileName, buf[0] != 0xFD);
+ if (result == 0) return 1;
+ filesize += result;
+#else
+ DISPLAYLEVEL(1, "zstd: %s: xz/lzma file cannot be uncompressed (zstd compiled without ZSTD_LZMADECOMPRESS) -- ignored \n", srcFileName);
+ return 1;
+#endif
+ } else {
+ if (!ZSTD_isFrame(ress.srcBuffer, toRead)) {
+ if ((g_overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */
+ unsigned const result = FIO_passThrough(ress.dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize, ress.srcBufferLoaded);
+ if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName); /* error should never happen */
+ return result;
+ } else {
+ DISPLAYLEVEL(1, "zstd: %s: not in zstd format \n", srcFileName);
+ fclose(srcFile);
+ return 1;
+ } }
+ filesize += FIO_decompressFrame(&ress, srcFile, filesize);
+ }
+ }
+
+ /* Final Status */
+ DISPLAYLEVEL(2, "\r%79s\r", "");
+ DISPLAYLEVEL(2, "%-20s: %llu bytes \n", srcFileName, filesize);
+
+ /* Close file */
+ if (fclose(srcFile)) EXM_THROW(33, "zstd: %s close error", srcFileName); /* error should never happen */
+ if (g_removeSrcFile /* --rm */ && strcmp(srcFileName, stdinmark)) { if (remove(srcFileName)) EXM_THROW(34, "zstd: %s: %s", srcFileName, strerror(errno)); };
+ return 0;
+}
+
+
+/** FIO_decompressFile_extRess() :
+ decompress `srcFileName` into `dstFileName`
+ @return : 0 : OK
+ 1 : operation aborted (src not available, dst already taken, etc.)
+*/
+static int FIO_decompressDstFile(dRess_t ress,
+ const char* dstFileName, const char* srcFileName)
+{
+ int result;
+ stat_t statbuf;
+ int stat_result = 0;
+
+ ress.dstFile = FIO_openDstFile(dstFileName);
+ if (ress.dstFile==0) return 1;
+
+ if (strcmp (srcFileName, stdinmark) && UTIL_getFileStat(srcFileName, &statbuf)) stat_result = 1;
+ result = FIO_decompressSrcFile(ress, dstFileName, srcFileName);
+
+ if (fclose(ress.dstFile)) EXM_THROW(38, "Write error : cannot properly close %s", dstFileName);
+
+ if ( (result != 0)
+ && strcmp(dstFileName, nulmark) /* special case : don't remove() /dev/null (#316) */
+ && remove(dstFileName) )
+ result=1; /* don't do anything special if remove() fails */
+ else if (strcmp (dstFileName, stdoutmark) && stat_result) UTIL_setFileStat(dstFileName, &statbuf);
+ return result;
+}
+
+
+int FIO_decompressFilename(const char* dstFileName, const char* srcFileName,
+ const char* dictFileName)
+{
+ int missingFiles = 0;
+ dRess_t ress = FIO_createDResources(dictFileName);
+
+ missingFiles += FIO_decompressDstFile(ress, dstFileName, srcFileName);
+
+ FIO_freeDResources(ress);
+ return missingFiles;
+}
+
+
+#define MAXSUFFIXSIZE 8
+int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
+ const char* suffix,
+ const char* dictFileName)
+{
+ int skippedFiles = 0;
+ int missingFiles = 0;
+ dRess_t ress = FIO_createDResources(dictFileName);
+
+ if (suffix==NULL) EXM_THROW(70, "zstd: decompression: unknown dst"); /* should never happen */
+
+ if (!strcmp(suffix, stdoutmark) || !strcmp(suffix, nulmark)) { /* special cases : -c or -t */
+ unsigned u;
+ ress.dstFile = FIO_openDstFile(suffix);
+ if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", suffix);
+ for (u=0; u<nbFiles; u++)
+ missingFiles += FIO_decompressSrcFile(ress, suffix, srcNamesTable[u]);
+ if (fclose(ress.dstFile)) EXM_THROW(72, "Write error : cannot properly close stdout");
+ } else {
+ size_t suffixSize;
+ size_t dfnSize = FNSPACE;
+ unsigned u;
+ char* dstFileName = (char*)malloc(FNSPACE);
+ if (dstFileName==NULL) EXM_THROW(73, "not enough memory for dstFileName");
+ for (u=0; u<nbFiles; u++) { /* create dstFileName */
+ const char* const srcFileName = srcNamesTable[u];
+ const char* const suffixPtr = strrchr(srcFileName, '.');
+ size_t const sfnSize = strlen(srcFileName);
+ if (!suffixPtr) {
+ DISPLAYLEVEL(1, "zstd: %s: unknown suffix -- ignored \n", srcFileName);
+ skippedFiles++;
+ continue;
+ }
+ suffixSize = strlen(suffixPtr);
+ if (dfnSize+suffixSize <= sfnSize+1) {
+ free(dstFileName);
+ dfnSize = sfnSize + 20;
+ dstFileName = (char*)malloc(dfnSize);
+ if (dstFileName==NULL) EXM_THROW(74, "not enough memory for dstFileName");
+ }
+ if (sfnSize <= suffixSize || (strcmp(suffixPtr, GZ_EXTENSION) && strcmp(suffixPtr, XZ_EXTENSION) && strcmp(suffixPtr, ZSTD_EXTENSION) && strcmp(suffixPtr, LZMA_EXTENSION))) {
+ DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%s/%s/%s/%s expected) -- ignored \n", srcFileName, GZ_EXTENSION, XZ_EXTENSION, ZSTD_EXTENSION, LZMA_EXTENSION);
+ skippedFiles++;
+ continue;
+ } else {
+ memcpy(dstFileName, srcFileName, sfnSize - suffixSize);
+ dstFileName[sfnSize-suffixSize] = '\0';
+ }
+
+ missingFiles += FIO_decompressDstFile(ress, dstFileName, srcFileName);
+ }
+ free(dstFileName);
+ }
+
+ FIO_freeDResources(ress);
+ return missingFiles + skippedFiles;
+}
+
+#endif /* #ifndef ZSTD_NODECOMPRESS */
diff --git a/programs/fileio.h b/programs/fileio.h
new file mode 100644
index 000000000000..0dd58d625d44
--- /dev/null
+++ b/programs/fileio.h
@@ -0,0 +1,94 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+
+#ifndef FILEIO_H_23981798732
+#define FILEIO_H_23981798732
+
+#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
+#include "zstd.h" /* ZSTD_* */
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+/* *************************************
+* Special i/o constants
+**************************************/
+#define stdinmark "/*stdin*\\"
+#define stdoutmark "/*stdout*\\"
+#ifdef _WIN32
+# define nulmark "nul"
+#else
+# define nulmark "/dev/null"
+#endif
+#define LZMA_EXTENSION ".lzma"
+#define XZ_EXTENSION ".xz"
+#define GZ_EXTENSION ".gz"
+#define ZSTD_EXTENSION ".zst"
+
+
+/*-*************************************
+* Types
+***************************************/
+typedef enum { FIO_zstdCompression, FIO_gzipCompression, FIO_xzCompression, FIO_lzmaCompression } FIO_compressionType_t;
+
+
+/*-*************************************
+* Parameters
+***************************************/
+void FIO_setCompressionType(FIO_compressionType_t compressionType);
+void FIO_overwriteMode(void);
+void FIO_setNotificationLevel(unsigned level);
+void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
+void FIO_setDictIDFlag(unsigned dictIDFlag);
+void FIO_setChecksumFlag(unsigned checksumFlag);
+void FIO_setRemoveSrcFile(unsigned flag);
+void FIO_setMemLimit(unsigned memLimit);
+void FIO_setNbThreads(unsigned nbThreads);
+void FIO_setBlockSize(unsigned blockSize);
+void FIO_setOverlapLog(unsigned overlapLog);
+
+
+/*-*************************************
+* Single File functions
+***************************************/
+/** FIO_compressFilename() :
+ @return : 0 == ok; 1 == pb with src file. */
+int FIO_compressFilename (const char* outfilename, const char* infilename, const char* dictFileName,
+ int compressionLevel, ZSTD_compressionParameters* comprParams);
+
+/** FIO_decompressFilename() :
+ @return : 0 == ok; 1 == pb with src file. */
+int FIO_decompressFilename (const char* outfilename, const char* infilename, const char* dictFileName);
+
+
+/*-*************************************
+* Multiple File functions
+***************************************/
+/** FIO_compressMultipleFilenames() :
+ @return : nb of missing files */
+int FIO_compressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
+ const char* suffix,
+ const char* dictFileName, int compressionLevel,
+ ZSTD_compressionParameters* comprParams);
+
+/** FIO_decompressMultipleFilenames() :
+ @return : nb of missing or skipped files */
+int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
+ const char* suffix,
+ const char* dictFileName);
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* FILEIO_H_23981798732 */
diff --git a/programs/platform.h b/programs/platform.h
new file mode 100644
index 000000000000..89a9f6cd42a0
--- /dev/null
+++ b/programs/platform.h
@@ -0,0 +1,137 @@
+/**
+ * platform.h - compiler and OS detection
+ *
+ * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+#ifndef PLATFORM_H_MODULE
+#define PLATFORM_H_MODULE
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+
+/* **************************************
+* Compiler Options
+****************************************/
+#if defined(_MSC_VER)
+# define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */
+# define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before <io.h> and <windows.h> */
+# if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */
+# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */
+# endif
+#endif
+
+
+/* **************************************
+* Detect 64-bit OS
+* http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros
+****************************************/
+#if defined __ia64 || defined _M_IA64 /* Intel Itanium */ \
+ || defined __powerpc64__ || defined __ppc64__ || defined __PPC64__ /* POWER 64-bit */ \
+ || (defined __sparc && (defined __sparcv9 || defined __sparc_v9__ || defined __arch64__)) || defined __sparc64__ /* SPARC 64-bit */ \
+ || defined __x86_64__s || defined _M_X64 /* x86 64-bit */ \
+ || defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__ /* ARM 64-bit */ \
+ || (defined __mips && (__mips == 64 || __mips == 4 || __mips == 3)) /* MIPS 64-bit */ \
+ || defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || defined _ADDR64 /* Cray */ \
+ || (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */
+# if !defined(__64BIT__)
+# define __64BIT__ 1
+# endif
+#endif
+
+
+/* *********************************************************
+* Turn on Large Files support (>4GB) for 32-bit Linux/Unix
+***********************************************************/
+#if !defined(__64BIT__) || defined(__MINGW32__) /* No point defining Large file for 64 bit but MinGW-w64 requires it */
+# if !defined(_FILE_OFFSET_BITS)
+# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */
+# endif
+# if !defined(_LARGEFILE_SOURCE) /* obsolete macro, replaced with _FILE_OFFSET_BITS */
+# define _LARGEFILE_SOURCE 1 /* Large File Support extension (LFS) - fseeko, ftello */
+# endif
+# if defined(_AIX) || defined(__hpux)
+# define _LARGE_FILES /* Large file support on 32-bits AIX and HP-UX */
+# endif
+#endif
+
+
+/* ************************************************************
+* Detect POSIX version
+* PLATFORM_POSIX_VERSION = -1 for non-Unix e.g. Windows
+* PLATFORM_POSIX_VERSION = 0 for Unix-like non-POSIX
+* PLATFORM_POSIX_VERSION >= 1 is equal to found _POSIX_VERSION
+***************************************************************/
+#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) /* UNIX-like OS */ \
+ || defined(__midipix__) || defined(__VMS))
+# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1–2001 (SUSv3) conformant */ \
+ || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* BSD distros */
+# define PLATFORM_POSIX_VERSION 200112L
+# else
+# if defined(__linux__) || defined(__linux)
+# ifndef _POSIX_C_SOURCE
+# define _POSIX_C_SOURCE 200112L /* use feature test macro */
+# endif
+# endif
+# include <unistd.h> /* declares _POSIX_VERSION */
+# if defined(_POSIX_VERSION) /* POSIX compliant */
+# define PLATFORM_POSIX_VERSION _POSIX_VERSION
+# else
+# define PLATFORM_POSIX_VERSION 0
+# endif
+# endif
+#endif
+#if !defined(PLATFORM_POSIX_VERSION)
+# define PLATFORM_POSIX_VERSION -1
+#endif
+
+
+/*-*********************************************
+* Detect if isatty() and fileno() are available
+************************************************/
+#if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) || defined(__DJGPP__)
+# include <unistd.h> /* isatty */
+# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
+#elif defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
+# include <io.h> /* _isatty */
+# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
+#else
+# define IS_CONSOLE(stdStream) 0
+#endif
+
+
+/******************************
+* OS-specific Includes
+******************************/
+#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32)
+# include <fcntl.h> /* _O_BINARY */
+# include <io.h> /* _setmode, _fileno, _get_osfhandle */
+# if !defined(__DJGPP__)
+# include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */
+# include <winioctl.h> /* FSCTL_SET_SPARSE */
+# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; }
+# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); }
+# else
+# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY)
+# define SET_SPARSE_FILE_MODE(file)
+# endif
+#else
+# define SET_BINARY_MODE(file)
+# define SET_SPARSE_FILE_MODE(file)
+#endif
+
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* PLATFORM_H_MODULE */
diff --git a/programs/util.h b/programs/util.h
new file mode 100644
index 000000000000..59e19d027ccd
--- /dev/null
+++ b/programs/util.h
@@ -0,0 +1,473 @@
+/**
+ * util.h - utility functions
+ *
+ * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+#ifndef UTIL_H_MODULE
+#define UTIL_H_MODULE
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+
+/*-****************************************
+* Dependencies
+******************************************/
+#include "platform.h" /* PLATFORM_POSIX_VERSION */
+#include <stdlib.h> /* malloc */
+#include <stddef.h> /* size_t, ptrdiff_t */
+#include <stdio.h> /* fprintf */
+#include <sys/types.h> /* stat, utime */
+#include <sys/stat.h> /* stat */
+#if defined(_MSC_VER)
+# include <sys/utime.h> /* utime */
+# include <io.h> /* _chmod */
+#else
+# include <unistd.h> /* chown, stat */
+# include <utime.h> /* utime */
+#endif
+#include <time.h> /* time */
+#include <errno.h>
+#include "mem.h" /* U32, U64 */
+
+
+/* ************************************************************
+* Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW
+***************************************************************/
+#if defined(_MSC_VER) && (_MSC_VER >= 1400)
+# define UTIL_fseek _fseeki64
+#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */
+# define UTIL_fseek fseeko
+#elif defined(__MINGW32__) && defined(__MSVCRT__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS)
+# define UTIL_fseek fseeko64
+#else
+# define UTIL_fseek fseek
+#endif
+
+
+/*-****************************************
+* Sleep functions: Windows - Posix - others
+******************************************/
+#if defined(_WIN32)
+# include <windows.h>
+# define SET_REALTIME_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS)
+# define UTIL_sleep(s) Sleep(1000*s)
+# define UTIL_sleepMilli(milli) Sleep(milli)
+#elif PLATFORM_POSIX_VERSION >= 0 /* Unix-like operating system */
+# include <unistd.h>
+# include <sys/resource.h> /* setpriority */
+# include <time.h> /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */
+# if defined(PRIO_PROCESS)
+# define SET_REALTIME_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
+# else
+# define SET_REALTIME_PRIORITY /* disabled */
+# endif
+# define UTIL_sleep(s) sleep(s)
+# if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) || (PLATFORM_POSIX_VERSION >= 200112L) /* nanosleep requires POSIX.1-2001 */
+# define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); }
+# else
+# define UTIL_sleepMilli(milli) /* disabled */
+# endif
+#else
+# define SET_REALTIME_PRIORITY /* disabled */
+# define UTIL_sleep(s) /* disabled */
+# define UTIL_sleepMilli(milli) /* disabled */
+#endif
+
+
+/* *************************************
+* Constants
+***************************************/
+#define LIST_SIZE_INCREASE (8*1024)
+
+
+/*-****************************************
+* Compiler specifics
+******************************************/
+#if defined(__INTEL_COMPILER)
+# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced, useful with UTIL_STATIC */
+#endif
+#if defined(__GNUC__)
+# define UTIL_STATIC static __attribute__((unused))
+#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+# define UTIL_STATIC static inline
+#elif defined(_MSC_VER)
+# define UTIL_STATIC static __inline
+#else
+# define UTIL_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
+#endif
+
+
+/*-****************************************
+* Time functions
+******************************************/
+#if defined(_WIN32) /* Windows */
+ typedef LARGE_INTEGER UTIL_freq_t;
+ typedef LARGE_INTEGER UTIL_time_t;
+ UTIL_STATIC void UTIL_initTimer(UTIL_freq_t* ticksPerSecond) { if (!QueryPerformanceFrequency(ticksPerSecond)) fprintf(stderr, "ERROR: QueryPerformance not present\n"); }
+ UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { QueryPerformanceCounter(x); }
+ UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_freq_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; }
+ UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_freq_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; }
+#elif defined(__APPLE__) && defined(__MACH__)
+ #include <mach/mach_time.h>
+ typedef mach_timebase_info_data_t UTIL_freq_t;
+ typedef U64 UTIL_time_t;
+ UTIL_STATIC void UTIL_initTimer(UTIL_freq_t* rate) { mach_timebase_info(rate); }
+ UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { *x = mach_absolute_time(); }
+ UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_freq_t rate, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return (((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom))/1000ULL; }
+ UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_freq_t rate, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return ((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom); }
+#elif (PLATFORM_POSIX_VERSION >= 200112L)
+ #include <sys/times.h> /* times */
+ typedef U64 UTIL_freq_t;
+ typedef U64 UTIL_time_t;
+ UTIL_STATIC void UTIL_initTimer(UTIL_freq_t* ticksPerSecond) { *ticksPerSecond=sysconf(_SC_CLK_TCK); }
+ UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { struct tms junk; clock_t newTicks = (clock_t) times(&junk); (void)junk; *x = (UTIL_time_t)newTicks; }
+ UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_freq_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / ticksPerSecond; }
+ UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_freq_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / ticksPerSecond; }
+#else /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */
+ typedef clock_t UTIL_freq_t;
+ typedef clock_t UTIL_time_t;
+ UTIL_STATIC void UTIL_initTimer(UTIL_freq_t* ticksPerSecond) { *ticksPerSecond=0; }
+ UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { *x = clock(); }
+ UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_freq_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
+ UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_freq_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
+#endif
+
+
+/* returns time span in microseconds */
+UTIL_STATIC U64 UTIL_clockSpanMicro( UTIL_time_t clockStart, UTIL_freq_t ticksPerSecond )
+{
+ UTIL_time_t clockEnd;
+ UTIL_getTime(&clockEnd);
+ return UTIL_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd);
+}
+
+
+UTIL_STATIC void UTIL_waitForNextTick(UTIL_freq_t ticksPerSecond)
+{
+ UTIL_time_t clockStart, clockEnd;
+ UTIL_getTime(&clockStart);
+ do {
+ UTIL_getTime(&clockEnd);
+ } while (UTIL_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0);
+}
+
+
+
+/*-****************************************
+* File functions
+******************************************/
+#if defined(_MSC_VER)
+ #define chmod _chmod
+ typedef struct __stat64 stat_t;
+#else
+ typedef struct stat stat_t;
+#endif
+
+
+UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf)
+{
+ int res = 0;
+ struct utimbuf timebuf;
+
+ timebuf.actime = time(NULL);
+ timebuf.modtime = statbuf->st_mtime;
+ res += utime(filename, &timebuf); /* set access and modification times */
+
+#if !defined(_WIN32)
+ res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */
+#endif
+
+ res += chmod(filename, statbuf->st_mode & 07777); /* Copy file permissions */
+
+ errno = 0;
+ return -res; /* number of errors is returned */
+}
+
+
+UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf)
+{
+ int r;
+#if defined(_MSC_VER)
+ r = _stat64(infilename, statbuf);
+ if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */
+#else
+ r = stat(infilename, statbuf);
+ if (r || !S_ISREG(statbuf->st_mode)) return 0; /* No good... */
+#endif
+ return 1;
+}
+
+
+UTIL_STATIC int UTIL_isRegFile(const char* infilename)
+{
+ stat_t statbuf;
+ return UTIL_getFileStat(infilename, &statbuf); /* Only need to know whether it is a regular file */
+}
+
+
+UTIL_STATIC U32 UTIL_isDirectory(const char* infilename)
+{
+ int r;
+ stat_t statbuf;
+#if defined(_MSC_VER)
+ r = _stat64(infilename, &statbuf);
+ if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
+#else
+ r = stat(infilename, &statbuf);
+ if (!r && S_ISDIR(statbuf.st_mode)) return 1;
+#endif
+ return 0;
+}
+
+
+UTIL_STATIC U64 UTIL_getFileSize(const char* infilename)
+{
+ int r;
+#if defined(_MSC_VER)
+ struct __stat64 statbuf;
+ r = _stat64(infilename, &statbuf);
+ if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */
+#elif defined(__MINGW32__) && defined (__MSVCRT__)
+ struct _stati64 statbuf;
+ r = _stati64(infilename, &statbuf);
+ if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */
+#else
+ struct stat statbuf;
+ r = stat(infilename, &statbuf);
+ if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
+#endif
+ return (U64)statbuf.st_size;
+}
+
+
+UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles)
+{
+ U64 total = 0;
+ unsigned n;
+ for (n=0; n<nbFiles; n++)
+ total += UTIL_getFileSize(fileNamesTable[n]);
+ return total;
+}
+
+
+/*
+ * A modified version of realloc().
+ * If UTIL_realloc() fails the original block is freed.
+*/
+UTIL_STATIC void *UTIL_realloc(void *ptr, size_t size)
+{
+ void *newptr = realloc(ptr, size);
+ if (newptr) return newptr;
+ free(ptr);
+ return NULL;
+}
+
+
+#ifdef _WIN32
+# define UTIL_HAS_CREATEFILELIST
+
+UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd)
+{
+ char* path;
+ int dirLength, fnameLength, pathLength, nbFiles = 0;
+ WIN32_FIND_DATAA cFile;
+ HANDLE hFile;
+
+ dirLength = (int)strlen(dirName);
+ path = (char*) malloc(dirLength + 3);
+ if (!path) return 0;
+
+ memcpy(path, dirName, dirLength);
+ path[dirLength] = '\\';
+ path[dirLength+1] = '*';
+ path[dirLength+2] = 0;
+
+ hFile=FindFirstFileA(path, &cFile);
+ if (hFile == INVALID_HANDLE_VALUE) {
+ fprintf(stderr, "Cannot open directory '%s'\n", dirName);
+ return 0;
+ }
+ free(path);
+
+ do {
+ fnameLength = (int)strlen(cFile.cFileName);
+ path = (char*) malloc(dirLength + fnameLength + 2);
+ if (!path) { FindClose(hFile); return 0; }
+ memcpy(path, dirName, dirLength);
+ path[dirLength] = '\\';
+ memcpy(path+dirLength+1, cFile.cFileName, fnameLength);
+ pathLength = dirLength+1+fnameLength;
+ path[pathLength] = 0;
+ if (cFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
+ if (strcmp (cFile.cFileName, "..") == 0 ||
+ strcmp (cFile.cFileName, ".") == 0) continue;
+
+ nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd); /* Recursively call "UTIL_prepareFileList" with the new path. */
+ if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
+ }
+ else if ((cFile.dwFileAttributes & FILE_ATTRIBUTE_NORMAL) || (cFile.dwFileAttributes & FILE_ATTRIBUTE_ARCHIVE) || (cFile.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED)) {
+ if (*bufStart + *pos + pathLength >= *bufEnd) {
+ ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
+ *bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
+ *bufEnd = *bufStart + newListSize;
+ if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
+ }
+ if (*bufStart + *pos + pathLength < *bufEnd) {
+ strncpy(*bufStart + *pos, path, *bufEnd - (*bufStart + *pos));
+ *pos += pathLength + 1;
+ nbFiles++;
+ }
+ }
+ free(path);
+ } while (FindNextFileA(hFile, &cFile));
+
+ FindClose(hFile);
+ return nbFiles;
+}
+
+#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */
+# define UTIL_HAS_CREATEFILELIST
+# include <dirent.h> /* opendir, readdir */
+# include <string.h> /* strerror, memcpy */
+
+UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd)
+{
+ DIR *dir;
+ struct dirent *entry;
+ char* path;
+ int dirLength, fnameLength, pathLength, nbFiles = 0;
+
+ if (!(dir = opendir(dirName))) {
+ fprintf(stderr, "Cannot open directory '%s': %s\n", dirName, strerror(errno));
+ return 0;
+ }
+
+ dirLength = (int)strlen(dirName);
+ errno = 0;
+ while ((entry = readdir(dir)) != NULL) {
+ if (strcmp (entry->d_name, "..") == 0 ||
+ strcmp (entry->d_name, ".") == 0) continue;
+ fnameLength = (int)strlen(entry->d_name);
+ path = (char*) malloc(dirLength + fnameLength + 2);
+ if (!path) { closedir(dir); return 0; }
+ memcpy(path, dirName, dirLength);
+ path[dirLength] = '/';
+ memcpy(path+dirLength+1, entry->d_name, fnameLength);
+ pathLength = dirLength+1+fnameLength;
+ path[pathLength] = 0;
+
+ if (UTIL_isDirectory(path)) {
+ nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd); /* Recursively call "UTIL_prepareFileList" with the new path. */
+ if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
+ } else {
+ if (*bufStart + *pos + pathLength >= *bufEnd) {
+ ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
+ *bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
+ *bufEnd = *bufStart + newListSize;
+ if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
+ }
+ if (*bufStart + *pos + pathLength < *bufEnd) {
+ strncpy(*bufStart + *pos, path, *bufEnd - (*bufStart + *pos));
+ *pos += pathLength + 1;
+ nbFiles++;
+ }
+ }
+ free(path);
+ errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */
+ }
+
+ if (errno != 0) {
+ fprintf(stderr, "readdir(%s) error: %s\n", dirName, strerror(errno));
+ free(*bufStart);
+ *bufStart = NULL;
+ }
+ closedir(dir);
+ return nbFiles;
+}
+
+#else
+
+UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd)
+{
+ (void)bufStart; (void)bufEnd; (void)pos;
+ fprintf(stderr, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName);
+ return 0;
+}
+
+#endif /* #ifdef _WIN32 */
+
+/*
+ * UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories,
+ * and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb).
+ * After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer)
+ * In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called.
+ */
+UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned inputNamesNb, char** allocatedBuffer, unsigned* allocatedNamesNb)
+{
+ size_t pos;
+ unsigned i, nbFiles;
+ char* buf = (char*)malloc(LIST_SIZE_INCREASE);
+ char* bufend = buf + LIST_SIZE_INCREASE;
+ const char** fileTable;
+
+ if (!buf) return NULL;
+
+ for (i=0, pos=0, nbFiles=0; i<inputNamesNb; i++) {
+ if (!UTIL_isDirectory(inputNames[i])) {
+ size_t const len = strlen(inputNames[i]);
+ if (buf + pos + len >= bufend) {
+ ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
+ buf = (char*)UTIL_realloc(buf, newListSize);
+ bufend = buf + newListSize;
+ if (!buf) return NULL;
+ }
+ if (buf + pos + len < bufend) {
+ strncpy(buf + pos, inputNames[i], bufend - (buf + pos));
+ pos += len + 1;
+ nbFiles++;
+ }
+ } else {
+ nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend);
+ if (buf == NULL) return NULL;
+ } }
+
+ if (nbFiles == 0) { free(buf); return NULL; }
+
+ fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*));
+ if (!fileTable) { free(buf); return NULL; }
+
+ for (i=0, pos=0; i<nbFiles; i++) {
+ fileTable[i] = buf + pos;
+ pos += strlen(fileTable[i]) + 1;
+ }
+
+ if (buf + pos > bufend) { free(buf); free((void*)fileTable); return NULL; }
+
+ *allocatedBuffer = buf;
+ *allocatedNamesNb = nbFiles;
+
+ return fileTable;
+}
+
+
+UTIL_STATIC void UTIL_freeFileList(const char** filenameTable, char* allocatedBuffer)
+{
+ if (allocatedBuffer) free(allocatedBuffer);
+ if (filenameTable) free((void*)filenameTable);
+}
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* UTIL_H_MODULE */
diff --git a/programs/windres/generate_res.bat b/programs/windres/generate_res.bat
new file mode 100644
index 000000000000..53891050ba29
--- /dev/null
+++ b/programs/windres/generate_res.bat
@@ -0,0 +1,11 @@
+@echo off
+REM http://stackoverflow.com/questions/708238/how-do-i-add-an-icon-to-a-mingw-gcc-compiled-executable
+
+where /q windres.exe
+IF ERRORLEVEL 1 (
+ ECHO The windres.exe is missing. Ensure it is installed and placed in your PATH.
+ EXIT /B
+) ELSE (
+ windres.exe -I ../lib -I windres -i windres/zstd.rc -O coff -F pe-x86-64 -o windres/zstd64.res
+ windres.exe -I ../lib -I windres -i windres/zstd.rc -O coff -F pe-i386 -o windres/zstd32.res
+)
diff --git a/programs/windres/verrsrc.h b/programs/windres/verrsrc.h
new file mode 100644
index 000000000000..e282add025ac
--- /dev/null
+++ b/programs/windres/verrsrc.h
@@ -0,0 +1,8 @@
+/* minimal set of defines required to generate zstd.res from zstd.rc */
+
+#define VS_VERSION_INFO 1
+
+#define VS_FFI_FILEFLAGSMASK 0x0000003FL
+#define VOS_NT_WINDOWS32 0x00040004L
+#define VFT_DLL 0x00000002L
+#define VFT2_UNKNOWN 0x00000000L
diff --git a/programs/windres/zstd.rc b/programs/windres/zstd.rc
new file mode 100644
index 000000000000..4a8aef3b1392
--- /dev/null
+++ b/programs/windres/zstd.rc
@@ -0,0 +1,51 @@
+// Microsoft Visual C++ generated resource script.
+//
+
+#include "zstd.h" /* ZSTD_VERSION_STRING */
+#define APSTUDIO_READONLY_SYMBOLS
+#include "verrsrc.h"
+#undef APSTUDIO_READONLY_SYMBOLS
+
+
+#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
+LANGUAGE 9, 1
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// Version
+//
+
+VS_VERSION_INFO VERSIONINFO
+ FILEVERSION ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0
+ PRODUCTVERSION ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0
+ FILEFLAGSMASK VS_FFI_FILEFLAGSMASK
+#ifdef _DEBUG
+ FILEFLAGS VS_FF_DEBUG
+#else
+ FILEFLAGS 0x0L
+#endif
+ FILEOS VOS_NT_WINDOWS32
+ FILETYPE VFT_DLL
+ FILESUBTYPE VFT2_UNKNOWN
+BEGIN
+ BLOCK "StringFileInfo"
+ BEGIN
+ BLOCK "040904B0"
+ BEGIN
+ VALUE "CompanyName", "Yann Collet, Facebook, Inc."
+ VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm"
+ VALUE "FileVersion", ZSTD_VERSION_STRING
+ VALUE "InternalName", "zstd.exe"
+ VALUE "LegalCopyright", "Copyright (c) 2013-present, Yann Collet, Facebook, Inc."
+ VALUE "OriginalFilename", "zstd.exe"
+ VALUE "ProductName", "Zstandard"
+ VALUE "ProductVersion", ZSTD_VERSION_STRING
+ END
+ END
+ BLOCK "VarFileInfo"
+ BEGIN
+ VALUE "Translation", 0x0409, 1200
+ END
+END
+
+#endif
diff --git a/programs/windres/zstd32.res b/programs/windres/zstd32.res
new file mode 100644
index 000000000000..b5dd78db717c
--- /dev/null
+++ b/programs/windres/zstd32.res
Binary files differ
diff --git a/programs/windres/zstd64.res b/programs/windres/zstd64.res
new file mode 100644
index 000000000000..32aff55428f0
--- /dev/null
+++ b/programs/windres/zstd64.res
Binary files differ
diff --git a/programs/zstd.1 b/programs/zstd.1
new file mode 100644
index 000000000000..684fb868aa30
--- /dev/null
+++ b/programs/zstd.1
@@ -0,0 +1,408 @@
+\"
+\" zstd.1: This is a manual page for 'zstd' program. This file is part of the
+\" zstd <http://www.zstd.net/> project.
+\" Author: Yann Collet
+\"
+
+\" No hyphenation
+.hy 0
+.nr HY 0
+
+.TH zstd "1" "2015-08-22" "zstd" "User Commands"
+.SH NAME
+\fBzstd, unzstd, zstdcat\fR - Compress or decompress .zst files
+
+.SH SYNOPSIS
+.TP 5
+\fBzstd\fR [\fBOPTIONS\fR] [-|INPUT-FILE] [-o <OUTPUT-FILE>]
+.PP
+.B unzstd
+is equivalent to
+.BR "zstd \-d"
+.br
+.B zstdcat
+is equivalent to
+.BR "zstd \-dcf"
+.br
+
+.SH DESCRIPTION
+.PP
+\fBzstd\fR is a fast lossless compression algorithm
+and data compression tool,
+with command line syntax similar to \fB gzip (1) \fR and \fB xz (1) \fR .
+It is based on the \fBLZ77\fR family, with further FSE & huff0 entropy stages.
+\fBzstd\fR offers highly configurable compression speed,
+with fast modes at > 200 MB/s per core,
+and strong modes nearing lzma compression ratios.
+It also features a very fast decoder, with speeds > 500 MB/s per core.
+
+\fBzstd\fR command line syntax is generally similar to gzip,
+but features the following differences :
+ - Source files are preserved by default.
+ It's possible to remove them automatically by using \fB--rm\fR command.
+ - When compressing a single file, \fBzstd\fR displays progress notifications and result summary by default.
+ Use \fB-q\fR to turn them off
+
+.PP
+.B zstd
+compresses or decompresses each
+.I file
+according to the selected operation mode.
+If no
+.I files
+are given or
+.I file
+is
+.BR \- ,
+.B zstd
+reads from standard input and writes the processed data
+to standard output.
+.B zstd
+will refuse (display an error and skip the
+.IR file )
+to write compressed data to standard output if it is a terminal.
+Similarly,
+.B zstd
+will refuse to read compressed data
+from standard input if it is a terminal.
+
+.PP
+Unless
+.B \-\-stdout
+or
+.B \-o
+is specified,
+.I files
+are written to a new file whose name is derived from the source
+.I file
+name:
+.IP \(bu 3
+When compressing, the suffix
+.B .zst
+is appended to the source filename to get the target filename.
+.IP \(bu 3
+When decompressing, the
+.B .zst
+suffix is removed from the filename to get the target filename.
+
+.SS "Concatenation with .zst files"
+It is possible to concatenate
+.B .zst
+files as is.
+.B zstd
+will decompress such files as if they were a single
+.B .zst
+file.
+
+
+
+.SH OPTIONS
+
+.
+.SS "Integer suffixes and special values"
+In most places where an integer argument is expected,
+an optional suffix is supported to easily indicate large integers.
+There must be no space between the integer and the suffix.
+.TP
+.B KiB
+Multiply the integer by 1,024 (2^10).
+.BR Ki ,
+.BR K ,
+and
+.B KB
+are accepted as synonyms for
+.BR KiB .
+.TP
+.B MiB
+Multiply the integer by 1,048,576 (2^20).
+.BR Mi ,
+.BR M ,
+and
+.B MB
+are accepted as synonyms for
+.BR MiB .
+
+.
+.SS "Operation mode"
+If multiple operation mode options are given,
+the last one takes effect.
+.TP
+.BR \-z ", " \-\-compress
+Compress.
+This is the default operation mode when no operation mode option
+is specified and no other operation mode is implied from
+the command name (for example,
+.B unzstd
+implies
+.BR \-\-decompress ).
+.TP
+.BR \-d ", " \-\-decompress ", " \-\-uncompress
+Decompress.
+.TP
+.BR \-t ", " \-\-test
+Test the integrity of compressed
+.IR files .
+This option is equivalent to
+.B "\-\-decompress \-\-stdout"
+except that the decompressed data is discarded instead of being
+written to standard output.
+No files are created or removed.
+.TP
+.B \-b#
+ benchmark file(s) using compression level #
+.TP
+.B \--train FILEs
+ use FILEs as training set to create a dictionary. The training set should contain a lot of small files (> 100).
+
+.
+.SS "Operation modifiers"
+.TP
+.B \-#
+ # compression level [1-19] (default:3)
+.TP
+.BR \--ultra
+ unlocks high compression levels 20+ (maximum 22), using a lot more memory.
+Note that decompression will also require more memory when using these levels.
+.TP
+.B \-D file
+ use `file` as Dictionary to compress or decompress FILE(s)
+.TP
+.BR \--no-dictID
+ do not store dictionary ID within frame header (dictionary compression).
+ The decoder will have to rely on implicit knowledge about which dictionary to use,
+it won't be able to check if it's correct.
+.TP
+.B \-o file
+ save result into `file` (only possible with a single INPUT-FILE)
+.TP
+.BR \-f ", " --force
+ overwrite output without prompting
+.TP
+.BR \-c ", " --stdout
+ force write to standard output, even if it is the console
+.TP
+.BR \--[no-]sparse
+ enable / disable sparse FS support, to make files with many zeroes smaller on disk.
+ Creating sparse files may save disk space and speed up the decompression
+by reducing the amount of disk I/O.
+ default : enabled when output is into a file, and disabled when output is stdout.
+ This setting overrides default and can force sparse mode over stdout.
+.TP
+.BR \--rm
+ remove source file(s) after successful compression or decompression
+.TP
+.BR \-k ", " --keep
+ keep source file(s) after successful compression or decompression.
+ This is the default behavior.
+.TP
+.BR \-r
+ operate recursively on directories
+.TP
+.BR \-h/\-H ", " --help
+ display help/long help and exit
+.TP
+.BR \-V ", " --version
+ display Version number and exit
+.TP
+.BR \-v ", " --verbose
+ verbose mode
+.TP
+.BR \-q ", " --quiet
+ suppress warnings, interactivity and notifications.
+ specify twice to suppress errors too.
+.TP
+.BR \-C ", " --[no-]check
+ add integrity check computed from uncompressed data (default : enabled)
+.TP
+.BR \-t ", " --test
+ Test the integrity of compressed files. This option is equivalent to \fB--decompress --stdout > /dev/null\fR.
+ No files are created or removed.
+.TP
+.BR --
+ All arguments after -- are treated as files
+
+
+.SH DICTIONARY BUILDER
+.PP
+\fBzstd\fR offers \fIdictionary\fR compression, useful for very small files and messages.
+It's possible to train \fBzstd\fR with some samples, the result of which is saved into a file called `dictionary`.
+Then during compression and decompression, make reference to the same dictionary.
+It will improve compression ratio of small files.
+Typical gains range from ~10% (at 64KB) to x5 better (at <1KB).
+.TP
+.B \--train FILEs
+ use FILEs as training set to create a dictionary. The training set should contain a lot of small files (> 100),
+and weight typically 100x the target dictionary size (for example, 10 MB for a 100 KB dictionary)
+.TP
+.B \-o file
+ dictionary saved into `file` (default: dictionary)
+.TP
+.B \--maxdict #
+ limit dictionary to specified size (default : 112640)
+.TP
+.B \--dictID #
+ A dictionary ID is a locally unique ID that a decoder can use to verify it is using the right dictionary.
+ By default, zstd will create a 4-bytes random number ID.
+ It's possible to give a precise number instead.
+ Short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header,
+ and an ID < 65536 will only need 2 bytes. This compares favorably to 4 bytes default.
+ However, it's up to the dictionary manager to not assign twice the same ID to 2 different dictionaries.
+.TP
+.B \-s#
+ dictionary selectivity level (default: 9)
+ the smaller the value, the denser the dictionary, improving its efficiency but reducing its possible maximum size.
+.TP
+.B \--cover=k=#,d=#
+ Use alternate dictionary builder algorithm named cover with parameters \fIk\fR and \fId\fR with \fId\fR <= \fIk\fR.
+ Selects segments of size \fIk\fR with the highest score to put in the dictionary.
+ The score of a segment is computed by the sum of the frequencies of all the subsegments of of size \fId\fR.
+ Generally \fId\fR should be in the range [6, 24].
+ Good values for \fIk\fR vary widely based on the input data, but a safe range is [32, 2048].
+ Example: \fB--train --cover=k=64,d=8 FILEs\fR.
+.TP
+.B \--optimize-cover[=steps=#,k=#,d=#]
+ If \fIsteps\fR is not specified, the default value of 32 is used.
+ If \fIk\fR is not specified, \fIsteps\fR values in [16, 2048] are checked for each value of \fId\fR.
+ If \fId\fR is not specified, the values checked are [6, 8, ..., 16].
+
+ Runs the cover dictionary builder for each parameter set saves the optimal parameters and dictionary.
+ Prints the optimal parameters and writes the optimal dictionary to the output file.
+ Supports multithreading if \fBzstd\fR is compiled with threading support.
+
+ The parameter \fIk\fR is more sensitve than \fId\fR, and is faster to optimize over.
+ Suggested use is to run with a \fIsteps\fR <= 32 with neither \fIk\fR nor \fId\fR set.
+ Once it completes, use the value of \fId\fR it selects with a higher \fIsteps\fR (in the range [256, 1024]).
+ \fBzstd --train --optimize-cover FILEs
+ \fBzstd --train --optimize-cover=d=d,steps=512 FILEs
+.TP
+
+.SH BENCHMARK
+.TP
+.B \-b#
+ benchmark file(s) using compression level #
+.TP
+.B \-e#
+ benchmark file(s) using multiple compression levels, from -b# to -e# (included).
+.TP
+.B \-i#
+ minimum evaluation time, in seconds (default : 3s), benchmark mode only
+.TP
+.B \-B#
+ cut file into independent blocks of size # (default: no block)
+.B \--priority=rt
+ set process priority to real-time
+
+.SH ADVANCED COMPRESSION OPTIONS
+.TP
+.B \--zstd[=\fIoptions\fR]
+.PD
+\fBzstd\fR provides 22 predefined compression levels. The selected or default predefined compression level can be changed with advanced compression options.
+The \fIoptions\fR are provided as a comma-separated list. You may specify only the \fIoptions\fR you want to change and the rest will be taken from the selected or default compression level.
+The list of available \fIoptions\fR:
+.RS
+
+.TP
+.BI strategy= strat
+.PD 0
+.TP
+.BI strat= strat
+.PD
+Specify a strategy used by a match finder.
+.IP ""
+There are 8 strategies numbered from 0 to 7, from faster to stronger:
+0=ZSTD_fast, 1=ZSTD_dfast, 2=ZSTD_greedy, 3=ZSTD_lazy, 4=ZSTD_lazy2, 5=ZSTD_btlazy2, 6=ZSTD_btopt, 7=ZSTD_btopt2.
+.IP ""
+
+.TP
+.BI windowLog= wlog
+.PD 0
+.TP
+.BI wlog= wlog
+.PD
+Specify the maximum number of bits for a match distance.
+.IP ""
+The higher number of bits increases the chance to find a match what usually improves compression ratio.
+It also increases memory requirements for compressor and decompressor.
+.IP ""
+The minimum \fIwlog\fR is 10 (1 KiB) and the maximum is 25 (32 MiB) for 32-bit compilation and 27 (128 MiB) for 64-bit compilation.
+.IP ""
+
+.TP
+.BI hashLog= hlog
+.PD 0
+.TP
+.BI hlog= hlog
+.PD
+Specify the maximum number of bits for a hash table.
+.IP ""
+The bigger hash table causes less collisions what usually make compression faster but requires more memory during compression.
+.IP ""
+The minimum \fIhlog\fR is 6 (64 B) and the maximum is 25 (32 MiB) for 32-bit compilation and 27 (128 MiB) for 64-bit compilation.
+
+.TP
+.BI chainLog= clog
+.PD 0
+.TP
+.BI clog= clog
+.PD
+Specify the maximum number of bits for a hash chain or a binary tree.
+.IP ""
+The higher number of bits increases the chance to find a match what usually improves compression ratio.
+It also slows down compression speed and increases memory requirements for compression.
+This option is ignored for the ZSTD_fast strategy.
+.IP ""
+The minimum \fIclog\fR is 6 (64 B) and the maximum is 26 (64 MiB) for 32-bit compilation and 28 (256 MiB) for 64-bit compilation.
+.IP ""
+
+.TP
+.BI searchLog= slog
+.PD 0
+.TP
+.BI slog= slog
+.PD
+Specify the maximum number of searches in a hash chain or a binary tree using logarithmic scale.
+.IP ""
+The bigger number of searches increases the chance to find a match what usually improves compression ratio but decreases compression speed.
+.IP ""
+The minimum \fIslog\fR is 1 and the maximum is 24 for 32-bit compilation and 26 for 64-bit compilation.
+.IP ""
+
+.TP
+.BI searchLength= slen
+.PD 0
+.TP
+.BI slen= slen
+.PD
+Specify the minimum searched length of a match in a hash table.
+.IP ""
+The bigger search length usually decreases compression ratio but improves decompression speed.
+.IP ""
+The minimum \fIslen\fR is 3 and the maximum is 7.
+.IP ""
+
+.TP
+.BI targetLength= tlen
+.PD 0
+.TP
+.BI tlen= tlen
+.PD
+Specify the minimum match length that causes a match finder to interrupt searching of better matches.
+.IP ""
+The bigger minimum match length usually improves compression ratio but decreases compression speed.
+This option is used only with ZSTD_btopt and ZSTD_btopt2 strategies.
+.IP ""
+The minimum \fItlen\fR is 4 and the maximum is 999.
+.IP ""
+
+.PP
+.B An example
+.br
+The following parameters sets advanced compression options to predefined level 19 for files bigger than 256 KB:
+.IP ""
+\fB--zstd=\fRwindowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6
+
+.SH BUGS
+Report bugs at:- https://github.com/facebook/zstd/issues
+
+.SH AUTHOR
+Yann Collet
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
new file mode 100644
index 000000000000..ae49da7b1e72
--- /dev/null
+++ b/programs/zstdcli.c
@@ -0,0 +1,696 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+
+/*-************************************
+* Tuning parameters
+**************************************/
+#ifndef ZSTDCLI_CLEVEL_DEFAULT
+# define ZSTDCLI_CLEVEL_DEFAULT 3
+#endif
+
+#ifndef ZSTDCLI_CLEVEL_MAX
+# define ZSTDCLI_CLEVEL_MAX 19 /* when not using --ultra */
+#endif
+
+
+
+/*-************************************
+* Dependencies
+**************************************/
+#include "platform.h" /* IS_CONSOLE, PLATFORM_POSIX_VERSION */
+#include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */
+#include <string.h> /* strcmp, strlen */
+#include <errno.h> /* errno */
+#include "fileio.h"
+#ifndef ZSTD_NOBENCH
+# include "bench.h" /* BMK_benchFiles, BMK_SetNbSeconds */
+#endif
+#ifndef ZSTD_NODICT
+# include "dibio.h"
+#endif
+#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */
+#include "zstd.h" /* ZSTD_VERSION_STRING */
+
+
+/*-************************************
+* Constants
+**************************************/
+#define COMPRESSOR_NAME "zstd command line interface"
+#ifndef ZSTD_VERSION
+# define ZSTD_VERSION "v" ZSTD_VERSION_STRING
+#endif
+#define AUTHOR "Yann Collet"
+#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR
+
+#define ZSTD_UNZSTD "unzstd"
+#define ZSTD_CAT "zstdcat"
+#define ZSTD_GZ "gzip"
+#define ZSTD_GUNZIP "gunzip"
+#define ZSTD_GZCAT "gzcat"
+#define ZSTD_LZMA "lzma"
+#define ZSTD_XZ "xz"
+
+#define KB *(1 <<10)
+#define MB *(1 <<20)
+#define GB *(1U<<30)
+
+#define DEFAULT_DISPLAY_LEVEL 2
+
+static const char* g_defaultDictName = "dictionary";
+static const unsigned g_defaultMaxDictSize = 110 KB;
+static const int g_defaultDictCLevel = 3;
+static const unsigned g_defaultSelectivityLevel = 9;
+#define OVERLAP_LOG_DEFAULT 9999
+static U32 g_overlapLog = OVERLAP_LOG_DEFAULT;
+
+
+/*-************************************
+* Display Macros
+**************************************/
+#define DISPLAY(...) fprintf(displayOut, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+static FILE* displayOut;
+static unsigned displayLevel = DEFAULT_DISPLAY_LEVEL; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */
+
+
+/*-************************************
+* Command Line
+**************************************/
+static int usage(const char* programName)
+{
+ DISPLAY( "Usage :\n");
+ DISPLAY( " %s [args] [FILE(s)] [-o file]\n", programName);
+ DISPLAY( "\n");
+ DISPLAY( "FILE : a filename\n");
+ DISPLAY( " with no FILE, or when FILE is - , read standard input\n");
+ DISPLAY( "Arguments :\n");
+#ifndef ZSTD_NOCOMPRESS
+ DISPLAY( " -# : # compression level (1-%d, default:%d) \n", ZSTDCLI_CLEVEL_MAX, ZSTDCLI_CLEVEL_DEFAULT);
+#endif
+#ifndef ZSTD_NODECOMPRESS
+ DISPLAY( " -d : decompression \n");
+#endif
+ DISPLAY( " -D file: use `file` as Dictionary \n");
+ DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n");
+ DISPLAY( " -f : overwrite output without prompting \n");
+ DISPLAY( "--rm : remove source file(s) after successful de/compression \n");
+ DISPLAY( " -k : preserve source file(s) (default) \n");
+ DISPLAY( " -h/-H : display help/long help and exit\n");
+ return 0;
+}
+
+static int usage_advanced(const char* programName)
+{
+ DISPLAY(WELCOME_MESSAGE);
+ usage(programName);
+ DISPLAY( "\n");
+ DISPLAY( "Advanced arguments :\n");
+ DISPLAY( " -V : display Version number and exit\n");
+ DISPLAY( " -v : verbose mode; specify multiple times to increase log level (default:%d)\n", DEFAULT_DISPLAY_LEVEL);
+ DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");
+ DISPLAY( " -c : force write to standard output, even if it is the console\n");
+#ifdef UTIL_HAS_CREATEFILELIST
+ DISPLAY( " -r : operate recursively on directories \n");
+#endif
+#ifndef ZSTD_NOCOMPRESS
+ DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
+ DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n");
+ DISPLAY( "--[no-]check : integrity check (default:enabled) \n");
+#ifdef ZSTD_MULTITHREAD
+ DISPLAY( " -T# : use # threads for compression (default:1) \n");
+ DISPLAY( " -B# : select size of independent sections (default:0==automatic) \n");
+#endif
+#ifdef ZSTD_GZCOMPRESS
+ DISPLAY( "--format=gzip : compress files to the .gz format \n");
+#endif
+#ifdef ZSTD_LZMACOMPRESS
+ DISPLAY( "--format=xz : compress files to the .xz format \n");
+ DISPLAY( "--format=lzma : compress files to the .lzma format \n");
+#endif
+#endif
+#ifndef ZSTD_NODECOMPRESS
+ DISPLAY( "--test : test compressed file integrity \n");
+ DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n");
+#endif
+ DISPLAY( " -M# : Set a memory usage limit for decompression \n");
+ DISPLAY( "-- : All arguments after \"--\" are treated as files \n");
+#ifndef ZSTD_NODICT
+ DISPLAY( "\n");
+ DISPLAY( "Dictionary builder :\n");
+ DISPLAY( "--train ## : create a dictionary from a training set of files \n");
+ DISPLAY( "--cover=k=#,d=# : use the cover algorithm with parameters k and d \n");
+ DISPLAY( "--optimize-cover[=steps=#,k=#,d=#] : optimize cover parameters with optional parameters\n");
+ DISPLAY( " -o file : `file` is dictionary name (default: %s) \n", g_defaultDictName);
+ DISPLAY( "--maxdict ## : limit dictionary to specified size (default : %u) \n", g_defaultMaxDictSize);
+ DISPLAY( " -s# : dictionary selectivity level (default: %u)\n", g_defaultSelectivityLevel);
+ DISPLAY( "--dictID ## : force dictionary ID to specified value (default: random)\n");
+#endif
+#ifndef ZSTD_NOBENCH
+ DISPLAY( "\n");
+ DISPLAY( "Benchmark arguments :\n");
+ DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n");
+ DISPLAY( " -e# : test all compression levels from -bX to # (default: 1)\n");
+ DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s)\n");
+ DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n");
+ DISPLAY( "--priority=rt : set process priority to real-time\n");
+#endif
+ return 0;
+}
+
+static int badusage(const char* programName)
+{
+ DISPLAYLEVEL(1, "Incorrect parameters\n");
+ if (displayLevel >= 1) usage(programName);
+ return 1;
+}
+
+static void waitEnter(void)
+{
+ int unused;
+ DISPLAY("Press enter to continue...\n");
+ unused = getchar();
+ (void)unused;
+}
+
+/*! readU32FromChar() :
+ @return : unsigned integer value read from input in `char` format
+ allows and interprets K, KB, KiB, M, MB and MiB suffix.
+ Will also modify `*stringPtr`, advancing it to position where it stopped reading.
+ Note : function result can overflow if digit string > MAX_UINT */
+static unsigned readU32FromChar(const char** stringPtr)
+{
+ unsigned result = 0;
+ while ((**stringPtr >='0') && (**stringPtr <='9'))
+ result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
+ if ((**stringPtr=='K') || (**stringPtr=='M')) {
+ result <<= 10;
+ if (**stringPtr=='M') result <<= 10;
+ (*stringPtr)++ ;
+ if (**stringPtr=='i') (*stringPtr)++;
+ if (**stringPtr=='B') (*stringPtr)++;
+ }
+ return result;
+}
+
+/** longCommandWArg() :
+ * check if *stringPtr is the same as longCommand.
+ * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
+ * @return 0 and doesn't modify *stringPtr otherwise.
+ */
+static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
+{
+ size_t const comSize = strlen(longCommand);
+ int const result = !strncmp(*stringPtr, longCommand, comSize);
+ if (result) *stringPtr += comSize;
+ return result;
+}
+
+
+#ifndef ZSTD_NODICT
+/**
+ * parseCoverParameters() :
+ * reads cover parameters from *stringPtr (e.g. "--cover=smoothing=100,kmin=48,kstep=4,kmax=64,d=8") into *params
+ * @return 1 means that cover parameters were correct
+ * @return 0 in case of malformed parameters
+ */
+static unsigned parseCoverParameters(const char* stringPtr, COVER_params_t *params)
+{
+ memset(params, 0, sizeof(*params));
+ for (; ;) {
+ if (longCommandWArg(&stringPtr, "k=")) { params->k = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+ if (longCommandWArg(&stringPtr, "d=")) { params->d = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+ if (longCommandWArg(&stringPtr, "steps=")) { params->steps = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+ return 0;
+ }
+ if (stringPtr[0] != 0) return 0;
+ DISPLAYLEVEL(4, "k=%u\nd=%u\nsteps=%u\n", params->k, params->d, params->steps);
+ return 1;
+}
+#endif
+
+
+/** parseCompressionParameters() :
+ * reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6") into *params
+ * @return 1 means that compression parameters were correct
+ * @return 0 in case of malformed parameters
+ */
+static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressionParameters* params)
+{
+ for ( ; ;) {
+ if (longCommandWArg(&stringPtr, "windowLog=") || longCommandWArg(&stringPtr, "wlog=")) { params->windowLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+ if (longCommandWArg(&stringPtr, "chainLog=") || longCommandWArg(&stringPtr, "clog=")) { params->chainLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+ if (longCommandWArg(&stringPtr, "hashLog=") || longCommandWArg(&stringPtr, "hlog=")) { params->hashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+ if (longCommandWArg(&stringPtr, "searchLog=") || longCommandWArg(&stringPtr, "slog=")) { params->searchLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+ if (longCommandWArg(&stringPtr, "searchLength=") || longCommandWArg(&stringPtr, "slen=")) { params->searchLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+ if (longCommandWArg(&stringPtr, "targetLength=") || longCommandWArg(&stringPtr, "tlen=")) { params->targetLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+ if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)(1 + readU32FromChar(&stringPtr)); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+ if (longCommandWArg(&stringPtr, "overlapLog=") || longCommandWArg(&stringPtr, "ovlog=")) { g_overlapLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+ return 0;
+ }
+
+ if (stringPtr[0] != 0) return 0; /* check the end of string */
+ DISPLAYLEVEL(4, "windowLog=%d\nchainLog=%d\nhashLog=%d\nsearchLog=%d\n", params->windowLog, params->chainLog, params->hashLog, params->searchLog);
+ DISPLAYLEVEL(4, "searchLength=%d\ntargetLength=%d\nstrategy=%d\n", params->searchLength, params->targetLength, params->strategy);
+ return 1;
+}
+
+
+typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train } zstd_operation_mode;
+
+#define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
+
+int main(int argCount, const char* argv[])
+{
+ int argNb,
+ forceStdout=0,
+ main_pause=0,
+ nextEntryIsDictionary=0,
+ operationResult=0,
+ nextArgumentIsOutFileName=0,
+ nextArgumentIsMaxDict=0,
+ nextArgumentIsDictID=0,
+ nextArgumentsAreFiles=0,
+ ultra=0,
+ lastCommand = 0,
+ nbThreads = 1,
+ setRealTimePrio = 0;
+ unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
+ size_t blockSize = 0;
+ zstd_operation_mode operation = zom_compress;
+ ZSTD_compressionParameters compressionParams;
+ int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
+ int cLevelLast = 1;
+ unsigned recursive = 0;
+ unsigned memLimit = 0;
+ const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */
+ unsigned filenameIdx = 0;
+ const char* programName = argv[0];
+ const char* outFileName = NULL;
+ const char* dictFileName = NULL;
+ const char* suffix = ZSTD_EXTENSION;
+ unsigned maxDictSize = g_defaultMaxDictSize;
+ unsigned dictID = 0;
+ int dictCLevel = g_defaultDictCLevel;
+ unsigned dictSelect = g_defaultSelectivityLevel;
+#ifdef UTIL_HAS_CREATEFILELIST
+ const char** extendedFileList = NULL;
+ char* fileNamesBuf = NULL;
+ unsigned fileNamesNb;
+#endif
+#ifndef ZSTD_NODICT
+ COVER_params_t coverParams;
+ int cover = 0;
+#endif
+
+ /* init */
+ (void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */
+ (void)dictCLevel; (void)dictSelect; (void)dictID; (void)maxDictSize; /* not used when ZSTD_NODICT set */
+ (void)ultra; (void)cLevel; /* not used when ZSTD_NOCOMPRESS set */
+ (void)memLimit; /* not used when ZSTD_NODECOMPRESS set */
+ if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); }
+ filenameTable[0] = stdinmark;
+ displayOut = stderr;
+ /* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */
+ { size_t pos;
+ for (pos = (int)strlen(programName); pos > 0; pos--) { if (programName[pos] == '/') { pos++; break; } }
+ programName += pos;
+ }
+
+ /* preset behaviors */
+ if (!strcmp(programName, ZSTD_UNZSTD)) operation=zom_decompress;
+ if (!strcmp(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; displayLevel=1; }
+ if (!strcmp(programName, ZSTD_GZ)) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); FIO_setRemoveSrcFile(1); } /* behave like gzip */
+ if (!strcmp(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(1); } /* behave like gunzip */
+ if (!strcmp(programName, ZSTD_GZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; displayLevel=1; } /* behave like gzcat */
+ if (!strcmp(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); } /* behave like lzma */
+ if (!strcmp(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like xz */
+ memset(&compressionParams, 0, sizeof(compressionParams));
+
+ /* command switches */
+ for (argNb=1; argNb<argCount; argNb++) {
+ const char* argument = argv[argNb];
+ if(!argument) continue; /* Protection if argument empty */
+
+ if (nextArgumentsAreFiles==0) {
+ /* "-" means stdin/stdout */
+ if (!strcmp(argument, "-")){
+ if (!filenameIdx) {
+ filenameIdx=1, filenameTable[0]=stdinmark;
+ outFileName=stdoutmark;
+ displayLevel-=(displayLevel==2);
+ continue;
+ } }
+
+ /* Decode commands (note : aggregated commands are allowed) */
+ if (argument[0]=='-') {
+
+ if (argument[1]=='-') {
+ /* long commands (--long-word) */
+ if (!strcmp(argument, "--")) { nextArgumentsAreFiles=1; continue; } /* only file names allowed from now on */
+ if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; }
+ if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; }
+ if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; }
+ if (!strcmp(argument, "--force")) { FIO_overwriteMode(); continue; }
+ if (!strcmp(argument, "--version")) { displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); }
+ if (!strcmp(argument, "--help")) { displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); }
+ if (!strcmp(argument, "--verbose")) { displayLevel++; continue; }
+ if (!strcmp(argument, "--quiet")) { displayLevel--; continue; }
+ if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel-=(displayLevel==2); continue; }
+ if (!strcmp(argument, "--ultra")) { ultra=1; continue; }
+ if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
+ if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; }
+ if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
+ if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; }
+ if (!strcmp(argument, "--test")) { operation=zom_test; continue; }
+ if (!strcmp(argument, "--train")) { operation=zom_train; outFileName=g_defaultDictName; continue; }
+ if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; lastCommand=1; continue; }
+ if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; lastCommand=1; continue; }
+ if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
+ if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; }
+ if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
+ if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
+#ifdef ZSTD_GZCOMPRESS
+ if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); continue; }
+#endif
+#ifdef ZSTD_LZMACOMPRESS
+ if (!strcmp(argument, "--format=lzma")) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression); continue; }
+ if (!strcmp(argument, "--format=xz")) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); continue; }
+#endif
+
+ /* long commands with arguments */
+#ifndef ZSTD_NODICT
+ if (longCommandWArg(&argument, "--cover=")) {
+ cover=1; if (!parseCoverParameters(argument, &coverParams)) CLEAN_RETURN(badusage(programName));
+ continue;
+ }
+ if (longCommandWArg(&argument, "--optimize-cover")) {
+ cover=2;
+ /* Allow optional arguments following an = */
+ if (*argument == 0) { memset(&coverParams, 0, sizeof(coverParams)); }
+ else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); }
+ else if (!parseCoverParameters(argument, &coverParams)) { CLEAN_RETURN(badusage(programName)); }
+ continue;
+ }
+#endif
+ if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; }
+ if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; }
+ if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; }
+ if (longCommandWArg(&argument, "--block-size=")) { blockSize = readU32FromChar(&argument); continue; }
+ if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) CLEAN_RETURN(badusage(programName)); continue; }
+ /* fall-through, will trigger bad_usage() later on */
+ }
+
+ argument++;
+ while (argument[0]!=0) {
+ if (lastCommand) {
+ DISPLAY("error : command must be followed by argument \n");
+ CLEAN_RETURN(1);
+ }
+#ifndef ZSTD_NOCOMPRESS
+ /* compression Level */
+ if ((*argument>='0') && (*argument<='9')) {
+ dictCLevel = cLevel = readU32FromChar(&argument);
+ continue;
+ }
+#endif
+
+ switch(argument[0])
+ {
+ /* Display help */
+ case 'V': displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); /* Version Only */
+ case 'H':
+ case 'h': displayOut=stdout; CLEAN_RETURN(usage_advanced(programName));
+
+ /* Compress */
+ case 'z': operation=zom_compress; argument++; break;
+
+ /* Decoding */
+ case 'd':
+#ifndef ZSTD_NOBENCH
+ if (operation==zom_bench) { BMK_setDecodeOnlyMode(1); argument++; break; } /* benchmark decode (hidden option) */
+#endif
+ operation=zom_decompress; argument++; break;
+
+ /* Force stdout, even if stdout==console */
+ case 'c': forceStdout=1; outFileName=stdoutmark; argument++; break;
+
+ /* Use file content as dictionary */
+ case 'D': nextEntryIsDictionary = 1; lastCommand = 1; argument++; break;
+
+ /* Overwrite */
+ case 'f': FIO_overwriteMode(); forceStdout=1; argument++; break;
+
+ /* Verbose mode */
+ case 'v': displayLevel++; argument++; break;
+
+ /* Quiet mode */
+ case 'q': displayLevel--; argument++; break;
+
+ /* keep source file (default); for gzip/xz compatibility */
+ case 'k': FIO_setRemoveSrcFile(0); argument++; break;
+
+ /* Checksum */
+ case 'C': argument++; FIO_setChecksumFlag(2); break;
+
+ /* test compressed file */
+ case 't': operation=zom_test; argument++; break;
+
+ /* destination file name */
+ case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break;
+
+ /* limit decompression memory */
+ case 'M':
+ argument++;
+ memLimit = readU32FromChar(&argument);
+ break;
+
+#ifdef UTIL_HAS_CREATEFILELIST
+ /* recursive */
+ case 'r': recursive=1; argument++; break;
+#endif
+
+#ifndef ZSTD_NOBENCH
+ /* Benchmark */
+ case 'b':
+ operation=zom_bench;
+ argument++;
+ break;
+
+ /* range bench (benchmark only) */
+ case 'e':
+ /* compression Level */
+ argument++;
+ cLevelLast = readU32FromChar(&argument);
+ break;
+
+ /* Modify Nb Iterations (benchmark only) */
+ case 'i':
+ argument++;
+ bench_nbSeconds = readU32FromChar(&argument);
+ break;
+
+ /* cut input into blocks (benchmark only) */
+ case 'B':
+ argument++;
+ blockSize = readU32FromChar(&argument);
+ break;
+
+#endif /* ZSTD_NOBENCH */
+
+ /* nb of threads (hidden option) */
+ case 'T':
+ argument++;
+ nbThreads = readU32FromChar(&argument);
+ break;
+
+ /* Dictionary Selection level */
+ case 's':
+ argument++;
+ dictSelect = readU32FromChar(&argument);
+ break;
+
+ /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */
+ case 'p': argument++;
+#ifndef ZSTD_NOBENCH
+ if ((*argument>='0') && (*argument<='9')) {
+ BMK_setAdditionalParam(readU32FromChar(&argument));
+ } else
+#endif
+ main_pause=1;
+ break;
+ /* unknown command */
+ default : CLEAN_RETURN(badusage(programName));
+ }
+ }
+ continue;
+ } /* if (argument[0]=='-') */
+
+ if (nextArgumentIsMaxDict) {
+ nextArgumentIsMaxDict = 0;
+ lastCommand = 0;
+ maxDictSize = readU32FromChar(&argument);
+ continue;
+ }
+
+ if (nextArgumentIsDictID) {
+ nextArgumentIsDictID = 0;
+ lastCommand = 0;
+ dictID = readU32FromChar(&argument);
+ continue;
+ }
+
+ } /* if (nextArgumentIsAFile==0) */
+
+ if (nextEntryIsDictionary) {
+ nextEntryIsDictionary = 0;
+ lastCommand = 0;
+ dictFileName = argument;
+ continue;
+ }
+
+ if (nextArgumentIsOutFileName) {
+ nextArgumentIsOutFileName = 0;
+ lastCommand = 0;
+ outFileName = argument;
+ if (!strcmp(outFileName, "-")) outFileName = stdoutmark;
+ continue;
+ }
+
+ /* add filename to list */
+ filenameTable[filenameIdx++] = argument;
+ }
+
+ if (lastCommand) { DISPLAY("error : command must be followed by argument \n"); CLEAN_RETURN(1); } /* forgotten argument */
+
+ /* Welcome message (if verbose) */
+ DISPLAYLEVEL(3, WELCOME_MESSAGE);
+#ifdef _POSIX_C_SOURCE
+ DISPLAYLEVEL(4, "_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE);
+#endif
+#ifdef _POSIX_VERSION
+ DISPLAYLEVEL(4, "_POSIX_VERSION defined: %ldL\n", (long) _POSIX_VERSION);
+#endif
+#ifdef PLATFORM_POSIX_VERSION
+ DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION);
+#endif
+
+#ifdef UTIL_HAS_CREATEFILELIST
+ if (recursive) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */
+ extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb);
+ if (extendedFileList) {
+ unsigned u;
+ for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, extendedFileList[u]);
+ free((void*)filenameTable);
+ filenameTable = extendedFileList;
+ filenameIdx = fileNamesNb;
+ }
+ }
+#endif
+
+ /* Check if benchmark is selected */
+ if (operation==zom_bench) {
+#ifndef ZSTD_NOBENCH
+ BMK_setNotificationLevel(displayLevel);
+ BMK_setBlockSize(blockSize);
+ BMK_setNbThreads(nbThreads);
+ BMK_setNbSeconds(bench_nbSeconds);
+ BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams, setRealTimePrio);
+#endif
+ (void)bench_nbSeconds;
+ goto _end;
+ }
+
+ /* Check if dictionary builder is selected */
+ if (operation==zom_train) {
+#ifndef ZSTD_NODICT
+ if (cover) {
+ coverParams.nbThreads = nbThreads;
+ coverParams.compressionLevel = dictCLevel;
+ coverParams.notificationLevel = displayLevel;
+ coverParams.dictID = dictID;
+ DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, NULL, &coverParams, cover - 1);
+ } else {
+ ZDICT_params_t dictParams;
+ memset(&dictParams, 0, sizeof(dictParams));
+ dictParams.compressionLevel = dictCLevel;
+ dictParams.selectivityLevel = dictSelect;
+ dictParams.notificationLevel = displayLevel;
+ dictParams.dictID = dictID;
+ DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, &dictParams, NULL, 0);
+ }
+#endif
+ goto _end;
+ }
+
+ /* No input filename ==> use stdin and stdout */
+ filenameIdx += !filenameIdx; /* filenameTable[0] is stdin by default */
+ if (!strcmp(filenameTable[0], stdinmark) && !outFileName) outFileName = stdoutmark; /* when input is stdin, default output is stdout */
+
+ /* Check if input/output defined as console; trigger an error in this case */
+ if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) ) CLEAN_RETURN(badusage(programName));
+ if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && strcmp(filenameTable[0], stdinmark) && !(forceStdout && (operation==zom_decompress)))
+ CLEAN_RETURN(badusage(programName));
+
+ /* user-selected output filename, only possible with a single file */
+ if (outFileName && strcmp(outFileName,stdoutmark) && strcmp(outFileName,nulmark) && (filenameIdx>1)) {
+ DISPLAY("Too many files (%u) on the command line. \n", filenameIdx);
+ CLEAN_RETURN(filenameIdx);
+ }
+
+#ifndef ZSTD_NOCOMPRESS
+ /* check compression level limits */
+ { int const maxCLevel = ultra ? ZSTD_maxCLevel() : ZSTDCLI_CLEVEL_MAX;
+ if (cLevel > maxCLevel) {
+ DISPLAYLEVEL(2, "Warning : compression level higher than max, reduced to %i \n", maxCLevel);
+ cLevel = maxCLevel;
+ } }
+#endif
+
+ /* No status message in pipe mode (stdin - stdout) or multi-files mode */
+ if (!strcmp(filenameTable[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1;
+ if ((filenameIdx>1) & (displayLevel==2)) displayLevel=1;
+
+ /* IO Stream/File */
+ FIO_setNotificationLevel(displayLevel);
+ if (operation==zom_compress) {
+#ifndef ZSTD_NOCOMPRESS
+ FIO_setNbThreads(nbThreads);
+ FIO_setBlockSize((U32)blockSize);
+ if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(g_overlapLog);
+ if ((filenameIdx==1) && outFileName)
+ operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, &compressionParams);
+ else
+ operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : suffix, dictFileName, cLevel, &compressionParams);
+#else
+ DISPLAY("Compression not supported\n");
+#endif
+ } else { /* decompression or test */
+#ifndef ZSTD_NODECOMPRESS
+ if (operation==zom_test) { outFileName=nulmark; FIO_setRemoveSrcFile(0); } /* test mode */
+ FIO_setMemLimit(memLimit);
+ if (filenameIdx==1 && outFileName)
+ operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName);
+ else
+ operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName);
+#else
+ DISPLAY("Decompression not supported\n");
+#endif
+ }
+
+_end:
+ if (main_pause) waitEnter();
+#ifdef UTIL_HAS_CREATEFILELIST
+ if (extendedFileList)
+ UTIL_freeFileList(extendedFileList, fileNamesBuf);
+ else
+#endif
+ free((void*)filenameTable);
+ return operationResult;
+}
diff --git a/programs/zstdgrep b/programs/zstdgrep
new file mode 100755
index 000000000000..9f871c03f1aa
--- /dev/null
+++ b/programs/zstdgrep
@@ -0,0 +1,124 @@
+#!/bin/sh
+#
+# Copyright (c) 2003 Thomas Klausner.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+grep=grep
+zcat=zstdcat
+
+endofopts=0
+pattern_found=0
+grep_args=""
+hyphen=0
+silent=0
+
+prg=$(basename $0)
+
+# handle being called 'zegrep' or 'zfgrep'
+case ${prg} in
+ *zegrep)
+ grep_args="-E";;
+ *zfgrep)
+ grep_args="-F";;
+esac
+
+# skip all options and pass them on to grep taking care of options
+# with arguments, and if -e was supplied
+
+while [ $# -gt 0 -a ${endofopts} -eq 0 ]
+do
+ case $1 in
+ # from GNU grep-2.5.1 -- keep in sync!
+ -[ABCDXdefm])
+ if [ $# -lt 2 ]
+ then
+ echo "${prg}: missing argument for $1 flag" >&2
+ exit 1
+ fi
+ case $1 in
+ -e)
+ pattern="$2"
+ pattern_found=1
+ shift 2
+ break
+ ;;
+ *)
+ ;;
+ esac
+ grep_args="${grep_args} $1 $2"
+ shift 2
+ ;;
+ --)
+ shift
+ endofopts=1
+ ;;
+ -)
+ hyphen=1
+ shift
+ ;;
+ -h)
+ silent=1
+ shift
+ ;;
+ -*)
+ grep_args="${grep_args} $1"
+ shift
+ ;;
+ *)
+ # pattern to grep for
+ endofopts=1
+ ;;
+ esac
+done
+
+# if no -e option was found, take next argument as grep-pattern
+if [ ${pattern_found} -lt 1 ]
+then
+ if [ $# -ge 1 ]; then
+ pattern="$1"
+ shift
+ elif [ ${hyphen} -gt 0 ]; then
+ pattern="-"
+ else
+ echo "${prg}: missing pattern" >&2
+ exit 1
+ fi
+fi
+
+# call grep ...
+if [ $# -lt 1 ]
+then
+ # ... on stdin
+ ${zcat} -fq - | ${grep} ${grep_args} -- "${pattern}" -
+else
+ # ... on all files given on the command line
+ if [ ${silent} -lt 1 -a $# -gt 1 ]; then
+ grep_args="-H ${grep_args}"
+ fi
+ while [ $# -gt 0 ]
+ do
+ ${zcat} -fq -- "$1" | ${grep} --label="${1}" ${grep_args} -- "${pattern}" -
+ shift
+ done
+fi
+
+exit 0
diff --git a/programs/zstdless b/programs/zstdless
new file mode 100755
index 000000000000..893799e7d95a
--- /dev/null
+++ b/programs/zstdless
@@ -0,0 +1,2 @@
+#!/bin/sh
+zstdcat "$@" | less
generated by cgit v1.3 (git 2.53.0) at 2026-03-11 01:54:42 +0000