diff options
| author | Xin LI <delphij@FreeBSD.org> | 2026-04-09 04:48:08 +0000 |
|---|---|---|
| committer | Xin LI <delphij@FreeBSD.org> | 2026-04-09 04:48:08 +0000 |
| commit | a985740c213b37ce1e5dd51ddf45c08e4c65f287 (patch) | |
| tree | bfd97f3200290513fbc36ae41a9df963716e6fd7 /examples | |
| parent | 7e509d50854e3f2ba38be3a5e640088c010333af (diff) | |
Diffstat (limited to 'examples')
| -rw-r--r-- | examples/Makefile | 93 | ||||
| -rw-r--r-- | examples/README.md | 46 | ||||
| -rw-r--r-- | examples/common.h | 246 | ||||
| -rw-r--r-- | examples/dictionary_compression.c | 107 | ||||
| -rw-r--r-- | examples/dictionary_decompression.c | 99 | ||||
| -rw-r--r-- | examples/multiple_simple_compression.c | 116 | ||||
| -rw-r--r-- | examples/multiple_streaming_compression.c | 133 | ||||
| -rw-r--r-- | examples/simple_compression.c | 68 | ||||
| -rw-r--r-- | examples/simple_decompression.c | 65 | ||||
| -rw-r--r-- | examples/streaming_compression.c | 146 | ||||
| -rw-r--r-- | examples/streaming_compression_thread_pool.c | 180 | ||||
| -rw-r--r-- | examples/streaming_decompression.c | 100 | ||||
| -rw-r--r-- | examples/streaming_memory_usage.c | 137 |
13 files changed, 0 insertions, 1536 deletions
diff --git a/examples/Makefile b/examples/Makefile deleted file mode 100644 index 31f52d35763f..000000000000 --- a/examples/Makefile +++ /dev/null @@ -1,93 +0,0 @@ -# ################################################################ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under both the BSD-style license (found in the -# LICENSE file in the root directory of this source tree) and the GPLv2 (found -# in the COPYING file in the root directory of this source tree). -# You may select, at your option, one of the above-listed licenses. -# ################################################################ - -LIBDIR =../lib -CPPFLAGS += -I$(LIBDIR) -LIB = $(LIBDIR)/libzstd.a - - -.PHONY: default -default: all - -.PHONY: all -all: simple_compression simple_decompression \ - multiple_simple_compression\ - dictionary_compression dictionary_decompression \ - streaming_compression streaming_decompression \ - multiple_streaming_compression streaming_memory_usage - -$(LIB) : - $(MAKE) -C $(LIBDIR) libzstd.a - -simple_compression.o: common.h -simple_compression : $(LIB) - -simple_decompression.o: common.h -simple_decompression : $(LIB) - -multiple_simple_compression.o: common.h -multiple_simple_compression : $(LIB) - -dictionary_compression.o: common.h -dictionary_compression : $(LIB) - -dictionary_decompression.o: common.h -dictionary_decompression : $(LIB) - -streaming_compression.o: common.h -streaming_compression : $(LIB) - -multiple_streaming_compression.o: common.h -multiple_streaming_compression : $(LIB) - -streaming_decompression.o: common.h -streaming_decompression : $(LIB) - -streaming_memory_usage.o: common.h -streaming_memory_usage : $(LIB) - - -.PHONY:clean -clean: - @$(RM) core *.o tmp* result* *.zst \ - simple_compression simple_decompression \ - multiple_simple_compression \ - dictionary_compression dictionary_decompression \ - streaming_compression streaming_decompression \ - multiple_streaming_compression streaming_memory_usage - @echo Cleaning completed - -.PHONY:test -test: all - cp README.md tmp - cp Makefile tmp2 - @echo -- Simple compression tests - ./simple_compression tmp - ./simple_decompression tmp.zst - ./multiple_simple_compression *.c - ./streaming_decompression tmp.zst > /dev/null - @echo -- Streaming memory usage - ./streaming_memory_usage - @echo -- Streaming compression tests - ./streaming_compression tmp - ./streaming_decompression tmp.zst > /dev/null - @echo -- Edge cases detection - ! ./streaming_decompression tmp # invalid input, must fail - ! ./simple_decompression tmp # invalid input, must fail - touch tmpNull # create 0-size file - ./simple_compression tmpNull - ./simple_decompression tmpNull.zst # 0-size frame : must work - @echo -- Multiple streaming tests - ./multiple_streaming_compression *.c - @echo -- Dictionary compression tests - ./dictionary_compression tmp2 tmp README.md - ./dictionary_decompression tmp2.zst tmp.zst README.md - $(RM) tmp* *.zst - @echo tests completed diff --git a/examples/README.md b/examples/README.md deleted file mode 100644 index 0bff7ac19a29..000000000000 --- a/examples/README.md +++ /dev/null @@ -1,46 +0,0 @@ -Zstandard library : usage examples -================================== - -- [Simple compression](simple_compression.c) : - Compress a single file. - Introduces usage of : `ZSTD_compress()` - -- [Simple decompression](simple_decompression.c) : - Decompress a single file. - Only compatible with simple compression. - Result remains in memory. - Introduces usage of : `ZSTD_decompress()` - -- [Multiple simple compression](multiple_simple_compression.c) : - Compress multiple files (in simple mode) in a single command line. - Demonstrates memory preservation technique that - minimizes malloc()/free() calls by re-using existing resources. - Introduces usage of : `ZSTD_compressCCtx()` - -- [Streaming memory usage](streaming_memory_usage.c) : - Provides amount of memory used by streaming context. - Introduces usage of : `ZSTD_sizeof_CStream()` - -- [Streaming compression](streaming_compression.c) : - Compress a single file. - Introduces usage of : `ZSTD_compressStream()` - -- [Multiple Streaming compression](multiple_streaming_compression.c) : - Compress multiple files (in streaming mode) in a single command line. - Introduces memory usage preservation technique, - reducing impact of malloc()/free() and memset() by re-using existing resources. - -- [Streaming decompression](streaming_decompression.c) : - Decompress a single file compressed by zstd. - Compatible with both simple and streaming compression. - Result is sent to stdout. - Introduces usage of : `ZSTD_decompressStream()` - -- [Dictionary compression](dictionary_compression.c) : - Compress multiple files using the same dictionary. - Introduces usage of : `ZSTD_createCDict()` and `ZSTD_compress_usingCDict()` - -- [Dictionary decompression](dictionary_decompression.c) : - Decompress multiple files using the same dictionary. - Result remains in memory. - Introduces usage of : `ZSTD_createDDict()` and `ZSTD_decompress_usingDDict()` diff --git a/examples/common.h b/examples/common.h deleted file mode 100644 index 4873e877a7af..000000000000 --- a/examples/common.h +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -/* - * This header file has common utility functions used in examples. - */ -#ifndef COMMON_H -#define COMMON_H - -#include <stdlib.h> // malloc, free, exit -#include <stdio.h> // fprintf, perror, fopen, etc. -#include <string.h> // strerror -#include <errno.h> // errno -#include <sys/stat.h> // stat -#include <zstd.h> - - -/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */ -#if defined(__GNUC__) -# define UNUSED_ATTR __attribute__((unused)) -#else -# define UNUSED_ATTR -#endif - -#define HEADER_FUNCTION static UNUSED_ATTR - - -/* - * Define the returned error code from utility functions. - */ -typedef enum { - ERROR_fsize = 1, - ERROR_fopen = 2, - ERROR_fclose = 3, - ERROR_fread = 4, - ERROR_fwrite = 5, - ERROR_loadFile = 6, - ERROR_saveFile = 7, - ERROR_malloc = 8, - ERROR_largeFile = 9, -} COMMON_ErrorCode; - -/*! CHECK - * Check that the condition holds. If it doesn't print a message and die. - */ -#define CHECK(cond, ...) \ - do { \ - if (!(cond)) { \ - fprintf(stderr, \ - "%s:%d CHECK(%s) failed: ", \ - __FILE__, \ - __LINE__, \ - #cond); \ - fprintf(stderr, "" __VA_ARGS__); \ - fprintf(stderr, "\n"); \ - exit(1); \ - } \ - } while (0) - -/*! CHECK_ZSTD - * Check the zstd error code and die if an error occurred after printing a - * message. - */ -#define CHECK_ZSTD(fn) \ - do { \ - size_t const err = (fn); \ - CHECK(!ZSTD_isError(err), "%s", ZSTD_getErrorName(err)); \ - } while (0) - -/*! fsize_orDie() : - * Get the size of a given file path. - * - * @return The size of a given file path. - */ -HEADER_FUNCTION size_t fsize_orDie(const char *filename) -{ - struct stat st; - if (stat(filename, &st) != 0) { - /* error */ - perror(filename); - exit(ERROR_fsize); - } - - off_t const fileSize = st.st_size; - size_t const size = (size_t)fileSize; - /* 1. fileSize should be non-negative, - * 2. if off_t -> size_t type conversion results in discrepancy, - * the file size is too large for type size_t. - */ - if ((fileSize < 0) || (fileSize != (off_t)size)) { - fprintf(stderr, "%s : filesize too large \n", filename); - exit(ERROR_largeFile); - } - return size; -} - -/*! fopen_orDie() : - * Open a file using given file path and open option. - * - * @return If successful this function will return a FILE pointer to an - * opened file otherwise it sends an error to stderr and exits. - */ -HEADER_FUNCTION FILE* fopen_orDie(const char *filename, const char *instruction) -{ - FILE* const inFile = fopen(filename, instruction); - if (inFile) return inFile; - /* error */ - perror(filename); - exit(ERROR_fopen); -} - -/*! fclose_orDie() : - * Close an opened file using given FILE pointer. - */ -HEADER_FUNCTION void fclose_orDie(FILE* file) -{ - if (!fclose(file)) { return; }; - /* error */ - perror("fclose"); - exit(ERROR_fclose); -} - -/*! fread_orDie() : - * - * Read sizeToRead bytes from a given file, storing them at the - * location given by buffer. - * - * @return The number of bytes read. - */ -HEADER_FUNCTION size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file) -{ - size_t const readSize = fread(buffer, 1, sizeToRead, file); - if (readSize == sizeToRead) return readSize; /* good */ - if (feof(file)) return readSize; /* good, reached end of file */ - /* error */ - perror("fread"); - exit(ERROR_fread); -} - -/*! fwrite_orDie() : - * - * Write sizeToWrite bytes to a file pointed to by file, obtaining - * them from a location given by buffer. - * - * Note: This function will send an error to stderr and exit if it - * cannot write data to the given file pointer. - * - * @return The number of bytes written. - */ -HEADER_FUNCTION size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file) -{ - size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file); - if (writtenSize == sizeToWrite) return sizeToWrite; /* good */ - /* error */ - perror("fwrite"); - exit(ERROR_fwrite); -} - -/*! malloc_orDie() : - * Allocate memory. - * - * @return If successful this function returns a pointer to allo- - * cated memory. If there is an error, this function will send that - * error to stderr and exit. - */ -HEADER_FUNCTION void* malloc_orDie(size_t size) -{ - void* const buff = malloc(size); - if (buff) return buff; - /* error */ - perror("malloc"); - exit(ERROR_malloc); -} - -/*! loadFile_orDie() : - * load file into buffer (memory). - * - * Note: This function will send an error to stderr and exit if it - * cannot read data from the given file path. - * - * @return If successful this function will load file into buffer and - * return file size, otherwise it will printout an error to stderr and exit. - */ -HEADER_FUNCTION size_t loadFile_orDie(const char* fileName, void* buffer, size_t bufferSize) -{ - size_t const fileSize = fsize_orDie(fileName); - CHECK(fileSize <= bufferSize, "File too large!"); - - FILE* const inFile = fopen_orDie(fileName, "rb"); - size_t const readSize = fread(buffer, 1, fileSize, inFile); - if (readSize != (size_t)fileSize) { - fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno)); - exit(ERROR_fread); - } - fclose(inFile); /* can't fail, read only */ - return fileSize; -} - -/*! mallocAndLoadFile_orDie() : - * allocate memory buffer and then load file into it. - * - * Note: This function will send an error to stderr and exit if memory allocation - * fails or it cannot read data from the given file path. - * - * @return If successful this function will return buffer and bufferSize(=fileSize), - * otherwise it will printout an error to stderr and exit. - */ -HEADER_FUNCTION void* mallocAndLoadFile_orDie(const char* fileName, size_t* bufferSize) -{ - size_t const fileSize = fsize_orDie(fileName); - *bufferSize = fileSize; - void* const buffer = malloc_orDie(*bufferSize); - loadFile_orDie(fileName, buffer, *bufferSize); - return buffer; -} - -/*! saveFile_orDie() : - * - * Save buffSize bytes to a given file path, obtaining them from a location pointed - * to by buff. - * - * Note: This function will send an error to stderr and exit if it - * cannot write to a given file. - */ -HEADER_FUNCTION void saveFile_orDie(const char* fileName, const void* buff, size_t buffSize) -{ - FILE* const oFile = fopen_orDie(fileName, "wb"); - size_t const wSize = fwrite(buff, 1, buffSize, oFile); - if (wSize != (size_t)buffSize) { - fprintf(stderr, "fwrite: %s : %s \n", fileName, strerror(errno)); - exit(ERROR_fwrite); - } - if (fclose(oFile)) { - perror(fileName); - exit(ERROR_fclose); - } -} - -#endif diff --git a/examples/dictionary_compression.c b/examples/dictionary_compression.c deleted file mode 100644 index 83edc1cad9b7..000000000000 --- a/examples/dictionary_compression.c +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. -**/ - -/* This example deals with Dictionary compression, - * its counterpart is `examples/dictionary_decompression.c` . - * These examples presume that a dictionary already exists. - * The main method to create a dictionary is `zstd --train`, - * look at the CLI documentation for details. - * Another possible method is to employ dictionary training API, - * published in `lib/zdict.h` . -**/ - -#include <stdio.h> // printf -#include <stdlib.h> // free -#include <string.h> // memset, strcat -#include <zstd.h> // presumes zstd library is installed -#include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD() - -/* createDict() : -** `dictFileName` is supposed already created using `zstd --train` */ -static ZSTD_CDict* createCDict_orDie(const char* dictFileName, int cLevel) -{ - size_t dictSize; - printf("loading dictionary %s \n", dictFileName); - void* const dictBuffer = mallocAndLoadFile_orDie(dictFileName, &dictSize); - ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, cLevel); - CHECK(cdict != NULL, "ZSTD_createCDict() failed!"); - free(dictBuffer); - return cdict; -} - - -static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdict) -{ - size_t fSize; - void* const fBuff = mallocAndLoadFile_orDie(fname, &fSize); - size_t const cBuffSize = ZSTD_compressBound(fSize); - void* const cBuff = malloc_orDie(cBuffSize); - - /* Compress using the dictionary. - * This function writes the dictionary id, and content size into the header. - * But, it doesn't use a checksum. You can control these options using the - * advanced API: ZSTD_CCtx_setParameter(), ZSTD_CCtx_refCDict(), - * and ZSTD_compress2(). - */ - ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - CHECK(cctx != NULL, "ZSTD_createCCtx() failed!"); - size_t const cSize = ZSTD_compress_usingCDict(cctx, cBuff, cBuffSize, fBuff, fSize, cdict); - CHECK_ZSTD(cSize); - - saveFile_orDie(oname, cBuff, cSize); - - /* success */ - printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname); - - ZSTD_freeCCtx(cctx); /* never fails */ - free(fBuff); - free(cBuff); -} - - -static char* createOutFilename_orDie(const char* filename) -{ - size_t const inL = strlen(filename); - size_t const outL = inL + 5; - void* outSpace = malloc_orDie(outL); - memset(outSpace, 0, outL); - strcat(outSpace, filename); - strcat(outSpace, ".zst"); - return (char*)outSpace; -} - -int main(int argc, const char** argv) -{ - const char* const exeName = argv[0]; - int const cLevel = 3; - - if (argc<3) { - fprintf(stderr, "wrong arguments\n"); - fprintf(stderr, "usage:\n"); - fprintf(stderr, "%s [FILES] dictionary\n", exeName); - return 1; - } - - /* load dictionary only once */ - const char* const dictName = argv[argc-1]; - ZSTD_CDict* const dictPtr = createCDict_orDie(dictName, cLevel); - - int u; - for (u=1; u<argc-1; u++) { - const char* inFilename = argv[u]; - char* const outFilename = createOutFilename_orDie(inFilename); - compress(inFilename, outFilename, dictPtr); - free(outFilename); - } - - ZSTD_freeCDict(dictPtr); - printf("All %u files compressed. \n", argc-2); - return 0; -} diff --git a/examples/dictionary_decompression.c b/examples/dictionary_decompression.c deleted file mode 100644 index e6c999964a24..000000000000 --- a/examples/dictionary_decompression.c +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - - -#include <stdio.h> // printf -#include <stdlib.h> // free -#include <zstd.h> // presumes zstd library is installed -#include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD() - -/* createDict() : - `dictFileName` is supposed to have been created using `zstd --train` */ -static ZSTD_DDict* createDict_orDie(const char* dictFileName) -{ - size_t dictSize; - printf("loading dictionary %s \n", dictFileName); - void* const dictBuffer = mallocAndLoadFile_orDie(dictFileName, &dictSize); - ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictSize); - CHECK(ddict != NULL, "ZSTD_createDDict() failed!"); - free(dictBuffer); - return ddict; -} - -static void decompress(const char* fname, const ZSTD_DDict* ddict) -{ - size_t cSize; - void* const cBuff = mallocAndLoadFile_orDie(fname, &cSize); - /* Read the content size from the frame header. For simplicity we require - * that it is always present. By default, zstd will write the content size - * in the header when it is known. If you can't guarantee that the frame - * content size is always written into the header, either use streaming - * decompression, or ZSTD_decompressBound(). - */ - unsigned long long const rSize = ZSTD_getFrameContentSize(cBuff, cSize); - CHECK(rSize != ZSTD_CONTENTSIZE_ERROR, "%s: not compressed by zstd!", fname); - CHECK(rSize != ZSTD_CONTENTSIZE_UNKNOWN, "%s: original size unknown!", fname); - void* const rBuff = malloc_orDie((size_t)rSize); - - /* Check that the dictionary ID matches. - * If a non-zstd dictionary is used, then both will be zero. - * By default zstd always writes the dictionary ID into the frame. - * Zstd will check if there is a dictionary ID mismatch as well. - */ - unsigned const expectedDictID = ZSTD_getDictID_fromDDict(ddict); - unsigned const actualDictID = ZSTD_getDictID_fromFrame(cBuff, cSize); - CHECK(actualDictID == expectedDictID, - "DictID mismatch: expected %u got %u", - expectedDictID, - actualDictID); - - /* Decompress using the dictionary. - * If you need to control the decompression parameters, then use the - * advanced API: ZSTD_DCtx_setParameter(), ZSTD_DCtx_refDDict(), and - * ZSTD_decompressDCtx(). - */ - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - CHECK(dctx != NULL, "ZSTD_createDCtx() failed!"); - size_t const dSize = ZSTD_decompress_usingDDict(dctx, rBuff, rSize, cBuff, cSize, ddict); - CHECK_ZSTD(dSize); - /* When zstd knows the content size, it will error if it doesn't match. */ - CHECK(dSize == rSize, "Impossible because zstd will check this condition!"); - - /* success */ - printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize); - - ZSTD_freeDCtx(dctx); - free(rBuff); - free(cBuff); -} - - -int main(int argc, const char** argv) -{ - const char* const exeName = argv[0]; - - if (argc<3) { - printf("wrong arguments\n"); - printf("usage:\n"); - printf("%s [FILES] dictionary\n", exeName); - return 1; - } - - /* load dictionary only once */ - const char* const dictName = argv[argc-1]; - ZSTD_DDict* const dictPtr = createDict_orDie(dictName); - - int u; - for (u=1; u<argc-1; u++) decompress(argv[u], dictPtr); - - ZSTD_freeDDict(dictPtr); - printf("All %u files correctly decoded (in memory) \n", argc-2); - return 0; -} diff --git a/examples/multiple_simple_compression.c b/examples/multiple_simple_compression.c deleted file mode 100644 index bf77ca13317f..000000000000 --- a/examples/multiple_simple_compression.c +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#include <stdio.h> // printf -#include <stdlib.h> // free -#include <string.h> // memcpy, strlen -#include <zstd.h> // presumes zstd library is installed -#include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD() - -typedef struct { - void* fBuffer; - void* cBuffer; - size_t fBufferSize; - size_t cBufferSize; - ZSTD_CCtx* cctx; -} resources; - -/* - * allocate memory for buffers big enough to compress all files - * as well as memory for output file name (ofn) - */ -static resources createResources_orDie(int argc, const char** argv, char **ofn, size_t* ofnBufferLen) -{ - size_t maxFilenameLength=0; - size_t maxFileSize = 0; - - int argNb; - for (argNb = 1; argNb < argc; argNb++) { - const char* const filename = argv[argNb]; - size_t const filenameLength = strlen(filename); - size_t const fileSize = fsize_orDie(filename); - - if (filenameLength > maxFilenameLength) maxFilenameLength = filenameLength; - if (fileSize > maxFileSize) maxFileSize = fileSize; - } - - resources ress; - ress.fBufferSize = maxFileSize; - ress.cBufferSize = ZSTD_compressBound(maxFileSize); - - *ofnBufferLen = maxFilenameLength + 5; - *ofn = (char*)malloc_orDie(*ofnBufferLen); - ress.fBuffer = malloc_orDie(ress.fBufferSize); - ress.cBuffer = malloc_orDie(ress.cBufferSize); - ress.cctx = ZSTD_createCCtx(); - CHECK(ress.cctx != NULL, "ZSTD_createCCtx() failed!"); - return ress; -} - -static void freeResources(resources ress, char *outFilename) -{ - free(ress.fBuffer); - free(ress.cBuffer); - ZSTD_freeCCtx(ress.cctx); /* never fails */ - free(outFilename); -} - -/* compress with pre-allocated context (ZSTD_CCtx) and input/output buffers*/ -static void compressFile_orDie(resources ress, const char* fname, const char* oname) -{ - size_t fSize = loadFile_orDie(fname, ress.fBuffer, ress.fBufferSize); - - /* Compress using the context. - * If you need more control over parameters, use the advanced API: - * ZSTD_CCtx_setParameter(), and ZSTD_compress2(). - */ - size_t const cSize = ZSTD_compressCCtx(ress.cctx, ress.cBuffer, ress.cBufferSize, ress.fBuffer, fSize, 1); - CHECK_ZSTD(cSize); - - saveFile_orDie(oname, ress.cBuffer, cSize); - - /* success */ - printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname); -} - -int main(int argc, const char** argv) -{ - const char* const exeName = argv[0]; - - if (argc<2) { - printf("wrong arguments\n"); - printf("usage:\n"); - printf("%s FILE(s)\n", exeName); - return 1; - } - - /* memory allocation for outFilename and resources */ - char* outFilename; - size_t outFilenameBufferLen; - resources const ress = createResources_orDie(argc, argv, &outFilename, &outFilenameBufferLen); - - /* compress files with shared context, input and output buffers */ - int argNb; - for (argNb = 1; argNb < argc; argNb++) { - const char* const inFilename = argv[argNb]; - size_t const inFilenameLen = strlen(inFilename); - CHECK(inFilenameLen + 5 <= outFilenameBufferLen, "File name too long!"); - memcpy(outFilename, inFilename, inFilenameLen); - memcpy(outFilename+inFilenameLen, ".zst", 5); - compressFile_orDie(ress, inFilename, outFilename); - } - - /* free memory */ - freeResources(ress,outFilename); - - printf("compressed %i files \n", argc-1); - - return 0; -} diff --git a/examples/multiple_streaming_compression.c b/examples/multiple_streaming_compression.c deleted file mode 100644 index b12ad03dce14..000000000000 --- a/examples/multiple_streaming_compression.c +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - - -/* The objective of this example is to show of to compress multiple successive files -* while preserving memory management. -* All structures and buffers will be created only once, -* and shared across all compression operations */ - -#include <stdio.h> // printf -#include <stdlib.h> // free -#include <string.h> // memset, strcat -#include <zstd.h> // presumes zstd library is installed -#include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD() - -typedef struct { - void* buffIn; - void* buffOut; - size_t buffInSize; - size_t buffOutSize; - ZSTD_CCtx* cctx; -} resources; - -static resources createResources_orDie(int cLevel) -{ - resources ress; - ress.buffInSize = ZSTD_CStreamInSize(); /* can always read one full block */ - ress.buffOutSize= ZSTD_CStreamOutSize(); /* can always flush a full block */ - ress.buffIn = malloc_orDie(ress.buffInSize); - ress.buffOut= malloc_orDie(ress.buffOutSize); - ress.cctx = ZSTD_createCCtx(); - CHECK(ress.cctx != NULL, "ZSTD_createCCtx() failed!"); - - /* Set any compression parameters you want here. - * They will persist for every compression operation. - * Here we set the compression level, and enable the checksum. - */ - CHECK_ZSTD( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, cLevel) ); - CHECK_ZSTD( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_checksumFlag, 1) ); - return ress; -} - -static void freeResources(resources ress) -{ - ZSTD_freeCCtx(ress.cctx); - free(ress.buffIn); - free(ress.buffOut); -} - -static void compressFile_orDie(resources ress, const char* fname, const char* outName) -{ - // Open the input and output files. - FILE* const fin = fopen_orDie(fname, "rb"); - FILE* const fout = fopen_orDie(outName, "wb"); - - /* Reset the context to a clean state to start a new compression operation. - * The parameters are sticky, so we keep the compression level and extra - * parameters that we set in createResources_orDie(). - */ - CHECK_ZSTD( ZSTD_CCtx_reset(ress.cctx, ZSTD_reset_session_only) ); - - size_t const toRead = ress.buffInSize; - size_t read; - while ( (read = fread_orDie(ress.buffIn, toRead, fin)) ) { - /* This loop is the same as streaming_compression.c. - * See that file for detailed comments. - */ - int const lastChunk = (read < toRead); - ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue; - - ZSTD_inBuffer input = { ress.buffIn, read, 0 }; - int finished; - do { - ZSTD_outBuffer output = { ress.buffOut, ress.buffOutSize, 0 }; - size_t const remaining = ZSTD_compressStream2(ress.cctx, &output, &input, mode); - CHECK_ZSTD(remaining); - fwrite_orDie(ress.buffOut, output.pos, fout); - finished = lastChunk ? (remaining == 0) : (input.pos == input.size); - } while (!finished); - CHECK(input.pos == input.size, - "Impossible: zstd only returns 0 when the input is completely consumed!"); - } - - fclose_orDie(fout); - fclose_orDie(fin); -} - -int main(int argc, const char** argv) -{ - const char* const exeName = argv[0]; - - if (argc<2) { - printf("wrong arguments\n"); - printf("usage:\n"); - printf("%s FILE(s)\n", exeName); - return 1; - } - - int const cLevel = 7; - resources const ress = createResources_orDie(cLevel); - void* ofnBuffer = NULL; - size_t ofnbSize = 0; - - int argNb; - for (argNb = 1; argNb < argc; argNb++) { - const char* const ifn = argv[argNb]; - size_t const ifnSize = strlen(ifn); - size_t const ofnSize = ifnSize + 5; - if (ofnbSize <= ofnSize) { - ofnbSize = ofnSize + 16; - free(ofnBuffer); - ofnBuffer = malloc_orDie(ofnbSize); - } - memset(ofnBuffer, 0, ofnSize); - strcat(ofnBuffer, ifn); - strcat(ofnBuffer, ".zst"); - compressFile_orDie(ress, ifn, ofnBuffer); - } - - freeResources(ress); - free(ofnBuffer); - - printf("compressed %i files \n", argc-1); - - return 0; -} diff --git a/examples/simple_compression.c b/examples/simple_compression.c deleted file mode 100644 index 7c880725fc76..000000000000 --- a/examples/simple_compression.c +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#include <stdio.h> // printf -#include <stdlib.h> // free -#include <string.h> // strlen, strcat, memset -#include <zstd.h> // presumes zstd library is installed -#include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD() - -static void compress_orDie(const char* fname, const char* oname) -{ - size_t fSize; - void* const fBuff = mallocAndLoadFile_orDie(fname, &fSize); - size_t const cBuffSize = ZSTD_compressBound(fSize); - void* const cBuff = malloc_orDie(cBuffSize); - - /* Compress. - * If you are doing many compressions, you may want to reuse the context. - * See the multiple_simple_compression.c example. - */ - size_t const cSize = ZSTD_compress(cBuff, cBuffSize, fBuff, fSize, 1); - CHECK_ZSTD(cSize); - - saveFile_orDie(oname, cBuff, cSize); - - /* success */ - printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname); - - free(fBuff); - free(cBuff); -} - -static char* createOutFilename_orDie(const char* filename) -{ - size_t const inL = strlen(filename); - size_t const outL = inL + 5; - void* const outSpace = malloc_orDie(outL); - memset(outSpace, 0, outL); - strcat(outSpace, filename); - strcat(outSpace, ".zst"); - return (char*)outSpace; -} - -int main(int argc, const char** argv) -{ - const char* const exeName = argv[0]; - - if (argc!=2) { - printf("wrong arguments\n"); - printf("usage:\n"); - printf("%s FILE\n", exeName); - return 1; - } - - const char* const inFilename = argv[1]; - - char* const outFilename = createOutFilename_orDie(inFilename); - compress_orDie(inFilename, outFilename); - free(outFilename); - return 0; -} diff --git a/examples/simple_decompression.c b/examples/simple_decompression.c deleted file mode 100644 index f499156f64ee..000000000000 --- a/examples/simple_decompression.c +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#include <stdio.h> // printf -#include <stdlib.h> // free -#include <zstd.h> // presumes zstd library is installed -#include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD() - -static void decompress(const char* fname) -{ - size_t cSize; - void* const cBuff = mallocAndLoadFile_orDie(fname, &cSize); - /* Read the content size from the frame header. For simplicity we require - * that it is always present. By default, zstd will write the content size - * in the header when it is known. If you can't guarantee that the frame - * content size is always written into the header, either use streaming - * decompression, or ZSTD_decompressBound(). - */ - unsigned long long const rSize = ZSTD_getFrameContentSize(cBuff, cSize); - CHECK(rSize != ZSTD_CONTENTSIZE_ERROR, "%s: not compressed by zstd!", fname); - CHECK(rSize != ZSTD_CONTENTSIZE_UNKNOWN, "%s: original size unknown!", fname); - - void* const rBuff = malloc_orDie((size_t)rSize); - - /* Decompress. - * If you are doing many decompressions, you may want to reuse the context - * and use ZSTD_decompressDCtx(). If you want to set advanced parameters, - * use ZSTD_DCtx_setParameter(). - */ - size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize); - CHECK_ZSTD(dSize); - /* When zstd knows the content size, it will error if it doesn't match. */ - CHECK(dSize == rSize, "Impossible because zstd will check this condition!"); - - /* success */ - printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize); - - free(rBuff); - free(cBuff); -} - -int main(int argc, const char** argv) -{ - const char* const exeName = argv[0]; - - if (argc!=2) { - printf("wrong arguments\n"); - printf("usage:\n"); - printf("%s FILE\n", exeName); - return 1; - } - - decompress(argv[1]); - - printf("%s correctly decoded (in memory). \n", argv[1]); - - return 0; -} diff --git a/examples/streaming_compression.c b/examples/streaming_compression.c deleted file mode 100644 index 063aa82a2946..000000000000 --- a/examples/streaming_compression.c +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - - -#include <stdio.h> // printf -#include <stdlib.h> // free -#include <string.h> // memset, strcat, strlen -#include <zstd.h> // presumes zstd library is installed -#include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD() - -static void compressFile_orDie(const char* fname, const char* outName, int cLevel, - int nbThreads) -{ - fprintf (stderr, "Starting compression of %s with level %d, using %d threads\n", - fname, cLevel, nbThreads); - - /* Open the input and output files. */ - FILE* const fin = fopen_orDie(fname, "rb"); - FILE* const fout = fopen_orDie(outName, "wb"); - /* Create the input and output buffers. - * They may be any size, but we recommend using these functions to size them. - * Performance will only suffer significantly for very tiny buffers. - */ - size_t const buffInSize = ZSTD_CStreamInSize(); - void* const buffIn = malloc_orDie(buffInSize); - size_t const buffOutSize = ZSTD_CStreamOutSize(); - void* const buffOut = malloc_orDie(buffOutSize); - - /* Create the context. */ - ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - CHECK(cctx != NULL, "ZSTD_createCCtx() failed!"); - - /* Set any parameters you want. - * Here we set the compression level, and enable the checksum. - */ - CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel) ); - CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) ); - if (nbThreads > 1) { - size_t const r = ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads); - if (ZSTD_isError(r)) { - fprintf (stderr, "Note: the linked libzstd library doesn't support multithreading. " - "Reverting to single-thread mode. \n"); - } - } - - /* This loop read from the input file, compresses that entire chunk, - * and writes all output produced to the output file. - */ - size_t const toRead = buffInSize; - for (;;) { - size_t read = fread_orDie(buffIn, toRead, fin); - /* Select the flush mode. - * If the read may not be finished (read == toRead) we use - * ZSTD_e_continue. If this is the last chunk, we use ZSTD_e_end. - * Zstd optimizes the case where the first flush mode is ZSTD_e_end, - * since it knows it is compressing the entire source in one pass. - */ - int const lastChunk = (read < toRead); - ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue; - /* Set the input buffer to what we just read. - * We compress until the input buffer is empty, each time flushing the - * output. - */ - ZSTD_inBuffer input = { buffIn, read, 0 }; - int finished; - do { - /* Compress into the output buffer and write all of the output to - * the file so we can reuse the buffer next iteration. - */ - ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; - size_t const remaining = ZSTD_compressStream2(cctx, &output , &input, mode); - CHECK_ZSTD(remaining); - fwrite_orDie(buffOut, output.pos, fout); - /* If we're on the last chunk we're finished when zstd returns 0, - * which means its consumed all the input AND finished the frame. - * Otherwise, we're finished when we've consumed all the input. - */ - finished = lastChunk ? (remaining == 0) : (input.pos == input.size); - } while (!finished); - CHECK(input.pos == input.size, - "Impossible: zstd only returns 0 when the input is completely consumed!"); - - if (lastChunk) { - break; - } - } - - ZSTD_freeCCtx(cctx); - fclose_orDie(fout); - fclose_orDie(fin); - free(buffIn); - free(buffOut); -} - - -static char* createOutFilename_orDie(const char* filename) -{ - size_t const inL = strlen(filename); - size_t const outL = inL + 5; - void* const outSpace = malloc_orDie(outL); - memset(outSpace, 0, outL); - strcat(outSpace, filename); - strcat(outSpace, ".zst"); - return (char*)outSpace; -} - -int main(int argc, const char** argv) -{ - const char* const exeName = argv[0]; - - if (argc < 2) { - printf("wrong arguments\n"); - printf("usage:\n"); - printf("%s FILE [LEVEL] [THREADS]\n", exeName); - return 1; - } - - int cLevel = 1; - int nbThreads = 1; - - if (argc >= 3) { - cLevel = atoi (argv[2]); - CHECK(cLevel != 0, "can't parse LEVEL!"); - } - - if (argc >= 4) { - nbThreads = atoi (argv[3]); - CHECK(nbThreads != 0, "can't parse THREADS!"); - } - - const char* const inFilename = argv[1]; - - char* const outFilename = createOutFilename_orDie(inFilename); - compressFile_orDie(inFilename, outFilename, cLevel, nbThreads); - - free(outFilename); /* not strictly required, since program execution stops there, - * but some static analyzer may complain otherwise */ - return 0; -} diff --git a/examples/streaming_compression_thread_pool.c b/examples/streaming_compression_thread_pool.c deleted file mode 100644 index a1a024129f2b..000000000000 --- a/examples/streaming_compression_thread_pool.c +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright (c) Martin Liska, SUSE, Meta Platforms, Inc. and affiliates. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - - -#include <stdio.h> // printf -#include <stdlib.h> // free -#include <string.h> // memset, strcat, strlen -#include <zstd.h> // presumes zstd library is installed -#include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD() -#include <pthread.h> - -typedef struct compress_args -{ - const char *fname; - char *outName; - int cLevel; -#if defined(ZSTD_STATIC_LINKING_ONLY) - ZSTD_threadPool *pool; -#endif -} compress_args_t; - -static void *compressFile_orDie(void *data) -{ - const int nbThreads = 16; - - compress_args_t *args = (compress_args_t *)data; - fprintf (stderr, "Starting compression of %s with level %d, using %d threads\n", args->fname, args->cLevel, nbThreads); - /* Open the input and output files. */ - FILE* const fin = fopen_orDie(args->fname, "rb"); - FILE* const fout = fopen_orDie(args->outName, "wb"); - /* Create the input and output buffers. - * They may be any size, but we recommend using these functions to size them. - * Performance will only suffer significantly for very tiny buffers. - */ - size_t const buffInSize = ZSTD_CStreamInSize(); - void* const buffIn = malloc_orDie(buffInSize); - size_t const buffOutSize = ZSTD_CStreamOutSize(); - void* const buffOut = malloc_orDie(buffOutSize); - - /* Create the context. */ - ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - CHECK(cctx != NULL, "ZSTD_createCCtx() failed!"); - -#if defined(ZSTD_STATIC_LINKING_ONLY) - size_t r = ZSTD_CCtx_refThreadPool(cctx, args->pool); - CHECK(r == 0, "ZSTD_CCtx_refThreadPool failed!"); -#endif - - /* Set any parameters you want. - * Here we set the compression level, and enable the checksum. - */ - CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, args->cLevel) ); - CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) ); - ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads); - - /* This loop reads from the input file, compresses that entire chunk, - * and writes all output produced to the output file. - */ - size_t const toRead = buffInSize; - for (;;) { - size_t read = fread_orDie(buffIn, toRead, fin); - /* Select the flush mode. - * If the read may not be finished (read == toRead) we use - * ZSTD_e_continue. If this is the last chunk, we use ZSTD_e_end. - * Zstd optimizes the case where the first flush mode is ZSTD_e_end, - * since it knows it is compressing the entire source in one pass. - */ - int const lastChunk = (read < toRead); - ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue; - /* Set the input buffer to what we just read. - * We compress until the input buffer is empty, each time flushing the - * output. - */ - ZSTD_inBuffer input = { buffIn, read, 0 }; - int finished; - do { - /* Compress into the output buffer and write all of the output to - * the file so we can reuse the buffer next iteration. - */ - ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; - size_t const remaining = ZSTD_compressStream2(cctx, &output , &input, mode); - CHECK_ZSTD(remaining); - fwrite_orDie(buffOut, output.pos, fout); - /* If we're on the last chunk we're finished when zstd returns 0, - * which means its consumed all the input AND finished the frame. - * Otherwise, we're finished when we've consumed all the input. - */ - finished = lastChunk ? (remaining == 0) : (input.pos == input.size); - } while (!finished); - CHECK(input.pos == input.size, - "Impossible: zstd only returns 0 when the input is completely consumed!"); - - if (lastChunk) { - break; - } - } - - fprintf (stderr, "Finishing compression of %s\n", args->outName); - - ZSTD_freeCCtx(cctx); - fclose_orDie(fout); - fclose_orDie(fin); - free(buffIn); - free(buffOut); - free(args->outName); - - return NULL; -} - - -static char* createOutFilename_orDie(const char* filename) -{ - size_t const inL = strlen(filename); - size_t const outL = inL + 5; - void* const outSpace = malloc_orDie(outL); - memset(outSpace, 0, outL); - strcat(outSpace, filename); - strcat(outSpace, ".zst"); - return (char*)outSpace; -} - -int main(int argc, const char** argv) -{ - const char* const exeName = argv[0]; - - if (argc<=3) { - printf("wrong arguments\n"); - printf("usage:\n"); - printf("%s POOL_SIZE LEVEL FILES\n", exeName); - return 1; - } - - int pool_size = atoi (argv[1]); - CHECK(pool_size != 0, "can't parse POOL_SIZE!"); - - int level = atoi (argv[2]); - CHECK(level != 0, "can't parse LEVEL!"); - - argc -= 3; - argv += 3; - -#if defined(ZSTD_STATIC_LINKING_ONLY) - ZSTD_threadPool *pool = ZSTD_createThreadPool (pool_size); - CHECK(pool != NULL, "ZSTD_createThreadPool() failed!"); - fprintf (stderr, "Using shared thread pool of size %d\n", pool_size); -#else - fprintf (stderr, "All threads use its own thread pool\n"); -#endif - - pthread_t *threads = malloc_orDie(argc * sizeof(pthread_t)); - compress_args_t *args = malloc_orDie(argc * sizeof(compress_args_t)); - - for (unsigned i = 0; i < argc; i++) - { - args[i].fname = argv[i]; - args[i].outName = createOutFilename_orDie(args[i].fname); - args[i].cLevel = level; -#if defined(ZSTD_STATIC_LINKING_ONLY) - args[i].pool = pool; -#endif - - pthread_create (&threads[i], NULL, compressFile_orDie, &args[i]); - } - - for (unsigned i = 0; i < argc; i++) - pthread_join (threads[i], NULL); - -#if defined(ZSTD_STATIC_LINKING_ONLY) - ZSTD_freeThreadPool (pool); -#endif - - return 0; -} diff --git a/examples/streaming_decompression.c b/examples/streaming_decompression.c deleted file mode 100644 index 95fa11227739..000000000000 --- a/examples/streaming_decompression.c +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - - -#include <stdio.h> // fprintf -#include <stdlib.h> // free -#include <zstd.h> // presumes zstd library is installed -#include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD() - -static void decompressFile_orDie(const char* fname) -{ - FILE* const fin = fopen_orDie(fname, "rb"); - size_t const buffInSize = ZSTD_DStreamInSize(); - void* const buffIn = malloc_orDie(buffInSize); - FILE* const fout = stdout; - size_t const buffOutSize = ZSTD_DStreamOutSize(); /* Guarantee to successfully flush at least one complete compressed block in all circumstances. */ - void* const buffOut = malloc_orDie(buffOutSize); - - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - CHECK(dctx != NULL, "ZSTD_createDCtx() failed!"); - - /* This loop assumes that the input file is one or more concatenated zstd - * streams. This example won't work if there is trailing non-zstd data at - * the end, but streaming decompression in general handles this case. - * ZSTD_decompressStream() returns 0 exactly when the frame is completed, - * and doesn't consume input after the frame. - */ - size_t const toRead = buffInSize; - size_t read; - size_t lastRet = 0; - int isEmpty = 1; - while ( (read = fread_orDie(buffIn, toRead, fin)) ) { - isEmpty = 0; - ZSTD_inBuffer input = { buffIn, read, 0 }; - /* Given a valid frame, zstd won't consume the last byte of the frame - * until it has flushed all of the decompressed data of the frame. - * Therefore, instead of checking if the return code is 0, we can - * decompress just check if input.pos < input.size. - */ - while (input.pos < input.size) { - ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; - /* The return code is zero if the frame is complete, but there may - * be multiple frames concatenated together. Zstd will automatically - * reset the context when a frame is complete. Still, calling - * ZSTD_DCtx_reset() can be useful to reset the context to a clean - * state, for instance if the last decompression call returned an - * error. - */ - size_t const ret = ZSTD_decompressStream(dctx, &output , &input); - CHECK_ZSTD(ret); - fwrite_orDie(buffOut, output.pos, fout); - lastRet = ret; - } - } - - if (isEmpty) { - fprintf(stderr, "input is empty\n"); - exit(1); - } - - if (lastRet != 0) { - /* The last return value from ZSTD_decompressStream did not end on a - * frame, but we reached the end of the file! We assume this is an - * error, and the input was truncated. - */ - fprintf(stderr, "EOF before end of stream: %zu\n", lastRet); - exit(1); - } - - ZSTD_freeDCtx(dctx); - fclose_orDie(fin); - fclose_orDie(fout); - free(buffIn); - free(buffOut); -} - - -int main(int argc, const char** argv) -{ - const char* const exeName = argv[0]; - - if (argc!=2) { - fprintf(stderr, "wrong arguments\n"); - fprintf(stderr, "usage:\n"); - fprintf(stderr, "%s FILE\n", exeName); - return 1; - } - - const char* const inFilename = argv[1]; - - decompressFile_orDie(inFilename); - return 0; -} diff --git a/examples/streaming_memory_usage.c b/examples/streaming_memory_usage.c deleted file mode 100644 index 957acb61a391..000000000000 --- a/examples/streaming_memory_usage.c +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - - -/*=== Tuning parameter ===*/ -#ifndef MAX_TESTED_LEVEL -#define MAX_TESTED_LEVEL 12 -#endif - - -/*=== Dependencies ===*/ -#include <stdio.h> // printf -#define ZSTD_STATIC_LINKING_ONLY -#include <zstd.h> // presumes zstd library is installed -#include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD() - - -/*=== functions ===*/ - -/*! readU32FromChar() : - @return : unsigned integer value read from input in `char` format - allows and interprets K, KB, KiB, M, MB and MiB suffix. - Will also modify `*stringPtr`, advancing it to position where it stopped reading. - Note : function result can overflow if digit string > MAX_UINT */ -static unsigned readU32FromChar(const char** stringPtr) -{ - unsigned result = 0; - while ((**stringPtr >='0') && (**stringPtr <='9')) - result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; - if ((**stringPtr=='K') || (**stringPtr=='M')) { - result <<= 10; - if (**stringPtr=='M') result <<= 10; - (*stringPtr)++ ; - if (**stringPtr=='i') (*stringPtr)++; - if (**stringPtr=='B') (*stringPtr)++; - } - return result; -} - - -int main(int argc, char const *argv[]) { - - printf("\n Zstandard (v%s) memory usage for streaming : \n\n", ZSTD_versionString()); - - unsigned wLog = 0; - if (argc > 1) { - const char* valStr = argv[1]; - wLog = readU32FromChar(&valStr); - } - - int compressionLevel; - for (compressionLevel = 1; compressionLevel <= MAX_TESTED_LEVEL; compressionLevel++) { -#define INPUT_SIZE 5 -#define COMPRESSED_SIZE 128 - char const dataToCompress[INPUT_SIZE] = "abcde"; - char compressedData[COMPRESSED_SIZE]; - char decompressedData[INPUT_SIZE]; - /* the ZSTD_CCtx_params structure is a way to save parameters and use - * them across multiple contexts. We use them here so we can call the - * function ZSTD_estimateCStreamSize_usingCCtxParams(). - */ - ZSTD_CCtx_params* const cctxParams = ZSTD_createCCtxParams(); - CHECK(cctxParams != NULL, "ZSTD_createCCtxParams() failed!"); - - /* Set the compression level. */ - CHECK_ZSTD( ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_compressionLevel, compressionLevel) ); - /* Set the window log. - * The value 0 means use the default window log, which is equivalent to - * not setting it. - */ - CHECK_ZSTD( ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, wLog) ); - - /* Force the compressor to allocate the maximum memory size for a given - * level by not providing the pledged source size, or calling - * ZSTD_compressStream2() with ZSTD_e_end. - */ - ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - CHECK(cctx != NULL, "ZSTD_createCCtx() failed!"); - CHECK_ZSTD( ZSTD_CCtx_setParametersUsingCCtxParams(cctx, cctxParams) ); - size_t compressedSize; - { - ZSTD_inBuffer inBuff = { dataToCompress, sizeof(dataToCompress), 0 }; - ZSTD_outBuffer outBuff = { compressedData, sizeof(compressedData), 0 }; - CHECK_ZSTD( ZSTD_compressStream(cctx, &outBuff, &inBuff) ); - size_t const remaining = ZSTD_endStream(cctx, &outBuff); - CHECK_ZSTD(remaining); - CHECK(remaining == 0, "Frame not flushed!"); - compressedSize = outBuff.pos; - } - - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - CHECK(dctx != NULL, "ZSTD_createDCtx() failed!"); - /* Set the maximum allowed window log. - * The value 0 means use the default window log, which is equivalent to - * not setting it. - */ - CHECK_ZSTD( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, wLog) ); - /* forces decompressor to use maximum memory size, since the - * decompressed size is not stored in the frame header. - */ - { ZSTD_inBuffer inBuff = { compressedData, compressedSize, 0 }; - ZSTD_outBuffer outBuff = { decompressedData, sizeof(decompressedData), 0 }; - size_t const remaining = ZSTD_decompressStream(dctx, &outBuff, &inBuff); - CHECK_ZSTD(remaining); - CHECK(remaining == 0, "Frame not complete!"); - CHECK(outBuff.pos == sizeof(dataToCompress), "Bad decompression!"); - } - - size_t const cstreamSize = ZSTD_sizeof_CStream(cctx); - size_t const cstreamEstimatedSize = ZSTD_estimateCStreamSize_usingCCtxParams(cctxParams); - size_t const dstreamSize = ZSTD_sizeof_DStream(dctx); - size_t const dstreamEstimatedSize = ZSTD_estimateDStreamSize_fromFrame(compressedData, compressedSize); - - CHECK(cstreamSize <= cstreamEstimatedSize, "Compression mem (%u) > estimated (%u)", - (unsigned)cstreamSize, (unsigned)cstreamEstimatedSize); - CHECK(dstreamSize <= dstreamEstimatedSize, "Decompression mem (%u) > estimated (%u)", - (unsigned)dstreamSize, (unsigned)dstreamEstimatedSize); - - printf("Level %2i : Compression Mem = %5u KB (estimated : %5u KB) ; Decompression Mem = %4u KB (estimated : %5u KB)\n", - compressionLevel, - (unsigned)(cstreamSize>>10), (unsigned)(cstreamEstimatedSize>>10), - (unsigned)(dstreamSize>>10), (unsigned)(dstreamEstimatedSize>>10)); - - ZSTD_freeDCtx(dctx); - ZSTD_freeCCtx(cctx); - ZSTD_freeCCtxParams(cctxParams); - if (wLog) break; /* single test */ - } - return 0; -} |
