diff options
author | Martin Matuska <mm@FreeBSD.org> | 2017-10-01 00:40:23 +0000 |
---|---|---|
committer | Martin Matuska <mm@FreeBSD.org> | 2017-10-01 00:40:23 +0000 |
commit | 5c831a5bd61576cacb48b39f8eeb47b92707a355 (patch) | |
tree | 315046b4d3237811245bd68cfa0099291b843129 /contrib/libarchive | |
parent | 0ffc7ed7e310f5810af6c04137678d0bef00f2f0 (diff) | |
parent | 533ab3c53efbfd230b718ed77dc9ea4d1446ddfc (diff) | |
download | src-5c831a5bd61576cacb48b39f8eeb47b92707a355.tar.gz src-5c831a5bd61576cacb48b39f8eeb47b92707a355.zip |
Notes
Diffstat (limited to 'contrib/libarchive')
49 files changed, 1494 insertions, 48 deletions
diff --git a/contrib/libarchive/cat/test/test_empty.zst.uu b/contrib/libarchive/cat/test/test_empty.zst.uu new file mode 100644 index 000000000000..44ae9a966c13 --- /dev/null +++ b/contrib/libarchive/cat/test/test_empty.zst.uu @@ -0,0 +1,4 @@ +begin 644 test_empty.zst +-*+4O_010`0``F>G840`` +` +end diff --git a/contrib/libarchive/cat/test/test_empty_zstd.c b/contrib/libarchive/cat/test/test_empty_zstd.c new file mode 100644 index 000000000000..c2d5ea106fa1 --- /dev/null +++ b/contrib/libarchive/cat/test/test_empty_zstd.c @@ -0,0 +1,41 @@ +/*- + * Copyright (c) 2017 Sean Purcell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "test.h" + +DEFINE_TEST(test_empty_zstd) +{ + const char *reffile = "test_empty.zst"; + int f; + + extract_reference_file(reffile); + f = systemf("%s %s >test.out 2>test.err", testprog, reffile); + if (f == 0 || canZstd()) { + assertEqualInt(0, f); + assertEmptyFile("test.out"); + assertEmptyFile("test.err"); + } else { + skipping("It seems zstd is not supported on this platform"); + } +} diff --git a/contrib/libarchive/cat/test/test_expand.zst.uu b/contrib/libarchive/cat/test/test_expand.zst.uu new file mode 100644 index 000000000000..89c28b1bc2f8 --- /dev/null +++ b/contrib/libarchive/cat/test/test_expand.zst.uu @@ -0,0 +1,4 @@ +begin 644 test_expand.zst +J*+4O_010Z0``8V]N=&5N=',@;V8@=&5S=%]E>'!A;F0N>G-T+@J;23#F +` +end diff --git a/contrib/libarchive/cat/test/test_expand_zstd.c b/contrib/libarchive/cat/test/test_expand_zstd.c new file mode 100644 index 000000000000..23d918252ad1 --- /dev/null +++ b/contrib/libarchive/cat/test/test_expand_zstd.c @@ -0,0 +1,41 @@ +/*- + * Copyright (c) 2017 Sean Purcell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "test.h" + +DEFINE_TEST(test_expand_zstd) +{ + const char *reffile = "test_expand.zst"; + int f; + + extract_reference_file(reffile); + f = systemf("%s %s >test.out 2>test.err", testprog, reffile); + if (f == 0 || canZstd()) { + assertEqualInt(0, f); + assertTextFileContents("contents of test_expand.zst.\n", "test.out"); + assertEmptyFile("test.err"); + } else { + skipping("It seems zstd is not supported on this platform"); + } +} diff --git a/contrib/libarchive/cpio/bsdcpio.1 b/contrib/libarchive/cpio/bsdcpio.1 index e52546e6f761..786a717097ed 100644 --- a/contrib/libarchive/cpio/bsdcpio.1 +++ b/contrib/libarchive/cpio/bsdcpio.1 @@ -187,6 +187,11 @@ In input mode, this option is ignored. Compress the archive with lz4-compatible compression before writing it. In input mode, this option is ignored; lz4 compression is recognized automatically on input. +.It Fl Fl zstd +(o mode only) +Compress the archive with zstd-compatible compression before writing it. +In input mode, this option is ignored; zstd compression is recognized +automatically on input. .It Fl Fl lzma (o mode only) Compress the file with lzma-compatible compression before writing it. diff --git a/contrib/libarchive/cpio/cmdline.c b/contrib/libarchive/cpio/cmdline.c index f37fe5e8431e..830caf659a07 100644 --- a/contrib/libarchive/cpio/cmdline.c +++ b/contrib/libarchive/cpio/cmdline.c @@ -92,6 +92,7 @@ static const struct option { { "verbose", 0, 'v' }, { "version", 0, OPTION_VERSION }, { "xz", 0, 'J' }, + { "zstd", 0, OPTION_ZSTD }, { NULL, 0, 0 } }; diff --git a/contrib/libarchive/cpio/cpio.c b/contrib/libarchive/cpio/cpio.c index cade829571bb..8396f1f98ce1 100644 --- a/contrib/libarchive/cpio/cpio.c +++ b/contrib/libarchive/cpio/cpio.c @@ -269,6 +269,7 @@ main(int argc, char *argv[]) case OPTION_LZ4: case OPTION_LZMA: /* GNU tar, others */ case OPTION_LZOP: /* GNU tar, others */ + case OPTION_ZSTD: cpio->compress = opt; break; case 'm': /* POSIX 1997 */ @@ -546,6 +547,9 @@ mode_out(struct cpio *cpio) case OPTION_LZOP: r = archive_write_add_filter_lzop(cpio->archive); break; + case OPTION_ZSTD: + r = archive_write_add_filter_zstd(cpio->archive); + break; case 'j': case 'y': r = archive_write_add_filter_bzip2(cpio->archive); break; diff --git a/contrib/libarchive/cpio/cpio.h b/contrib/libarchive/cpio/cpio.h index a383560bcf7b..b10f387be0e3 100644 --- a/contrib/libarchive/cpio/cpio.h +++ b/contrib/libarchive/cpio/cpio.h @@ -111,7 +111,8 @@ enum { OPTION_PRESERVE_OWNER, OPTION_QUIET, OPTION_UUENCODE, - OPTION_VERSION + OPTION_VERSION, + OPTION_ZSTD, }; int cpio_getopt(struct cpio *cpio); diff --git a/contrib/libarchive/cpio/test/test_extract.cpio.zst.uu b/contrib/libarchive/cpio/test/test_extract.cpio.zst.uu new file mode 100644 index 000000000000..5ec854b85d28 --- /dev/null +++ b/contrib/libarchive/cpio/test/test_extract.cpio.zst.uu @@ -0,0 +1,6 @@ +begin 644 test_extract.cpio.zst +M*+4O_01090,`,@41%X")&@#'G6T\K16_MR)#=DK)5:.1,2J0HY2"!(1!`!7R +M$(UB`2"*D41;J2UF&)<0!Y7X'TU<%W.\W^R]GO-WW^OO^QX0`%P<]30-!#U` +?!KD!`#XP,_`U4`HT3+RF:#!7Y\V@R)5"7P"^;WEUK@`` +` +end diff --git a/contrib/libarchive/cpio/test/test_extract_cpio_zstd.c b/contrib/libarchive/cpio/test/test_extract_cpio_zstd.c new file mode 100644 index 000000000000..289f33d0583c --- /dev/null +++ b/contrib/libarchive/cpio/test/test_extract_cpio_zstd.c @@ -0,0 +1,48 @@ +/*- + * Copyright (c) 2017 Sean Purcell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "test.h" +__FBSDID("$FreeBSD$"); + +DEFINE_TEST(test_extract_cpio_zstd) +{ + const char *reffile = "test_extract.cpio.zst"; + int f; + + extract_reference_file(reffile); + f = systemf("%s -it < %s >test.out 2>test.err", testprog, reffile); + if (f == 0 || canZstd()) { + assertEqualInt(0, systemf("%s -i < %s >test.out 2>test.err", + testprog, reffile)); + + assertFileExists("file1"); + assertTextFileContents("contents of file1.\n", "file1"); + assertFileExists("file2"); + assertTextFileContents("contents of file2.\n", "file2"); + assertEmptyFile("test.out"); + assertTextFileContents("1 block\n", "test.err"); + } else { + skipping("It seems zstd is not supported on this platform"); + } +} diff --git a/contrib/libarchive/cpio/test/test_option_zstd.c b/contrib/libarchive/cpio/test/test_option_zstd.c new file mode 100644 index 000000000000..29b8c78b94aa --- /dev/null +++ b/contrib/libarchive/cpio/test/test_option_zstd.c @@ -0,0 +1,85 @@ +/*- + * Copyright (c) 2017 Sean Purcell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "test.h" +__FBSDID("$FreeBSD$"); + +DEFINE_TEST(test_option_zstd) +{ + char *p; + int r; + size_t s; + + /* Create a file. */ + assertMakeFile("f", 0644, "a"); + + /* Archive it with zstd compression. */ + r = systemf("echo f | %s -o --zstd >archive.out 2>archive.err", + testprog); + p = slurpfile(&s, "archive.err"); + p[s] = '\0'; + if (r != 0) { + if (strstr(p, "Unsupported compression") != NULL) { + skipping("This version of bsdcpio was compiled " + "without zstd support"); + goto done; + } + /* POSIX permits different handling of the spawnp + * system call used to launch the subsidiary + * program: */ + /* Some systems fail immediately to spawn the new process. */ + if (strstr(p, "Can't launch") != NULL && !canZstd()) { + skipping("This version of bsdcpio uses an external zstd program " + "but no such program is available on this system."); + goto done; + } + /* Some systems successfully spawn the new process, + * but fail to exec a program within that process. + * This results in failure at the first attempt to + * write. */ + if (strstr(p, "Can't write") != NULL && !canZstd()) { + skipping("This version of bsdcpio uses an external zstd program " + "but no such program is available on this system."); + goto done; + } + /* On some systems the error won't be detected until closing + time, by a 127 exit error returned by waitpid. */ + if (strstr(p, "Error closing") != NULL && !canZstd()) { + skipping("This version of bsdcpio uses an external zstd program " + "but no such program is available on this system."); + return; + } + failure("--zstd option is broken: %s", p); + assertEqualInt(r, 0); + goto done; + } + free(p); + /* Check that the archive file has an zstd signature. */ + p = slurpfile(&s, "archive.out"); + assert(s > 2); + assertEqualMem(p, "\x28\xb5\x2f\xfd", 4); + +done: + free(p); +} diff --git a/contrib/libarchive/libarchive/archive.h b/contrib/libarchive/libarchive/archive.h index c3681e60c50a..46938ad20cdf 100644 --- a/contrib/libarchive/libarchive/archive.h +++ b/contrib/libarchive/libarchive/archive.h @@ -177,6 +177,7 @@ __LA_DECL const char * archive_zlib_version(void); __LA_DECL const char * archive_liblzma_version(void); __LA_DECL const char * archive_bzlib_version(void); __LA_DECL const char * archive_liblz4_version(void); +__LA_DECL const char * archive_libzstd_version(void); /* Declare our basic types. */ struct archive; @@ -276,6 +277,7 @@ typedef const char *archive_passphrase_callback(struct archive *, #define ARCHIVE_FILTER_LZOP 11 #define ARCHIVE_FILTER_GRZIP 12 #define ARCHIVE_FILTER_LZ4 13 +#define ARCHIVE_FILTER_ZSTD 14 #if ARCHIVE_VERSION_NUMBER < 4000000 #define ARCHIVE_COMPRESSION_NONE ARCHIVE_FILTER_NONE @@ -433,6 +435,7 @@ __LA_DECL int archive_read_support_filter_program_signature __LA_DECL int archive_read_support_filter_rpm(struct archive *); __LA_DECL int archive_read_support_filter_uu(struct archive *); __LA_DECL int archive_read_support_filter_xz(struct archive *); +__LA_DECL int archive_read_support_filter_zstd(struct archive *); __LA_DECL int archive_read_support_format_7zip(struct archive *); __LA_DECL int archive_read_support_format_all(struct archive *); @@ -778,6 +781,7 @@ __LA_DECL int archive_write_add_filter_program(struct archive *, const char *cmd); __LA_DECL int archive_write_add_filter_uuencode(struct archive *); __LA_DECL int archive_write_add_filter_xz(struct archive *); +__LA_DECL int archive_write_add_filter_zstd(struct archive *); /* A convenience function to set the format based on the code or name. */ diff --git a/contrib/libarchive/libarchive/archive_cmdline.c b/contrib/libarchive/libarchive/archive_cmdline.c index 7d3bac53bb5c..5c519cd17f1b 100644 --- a/contrib/libarchive/libarchive/archive_cmdline.c +++ b/contrib/libarchive/libarchive/archive_cmdline.c @@ -100,10 +100,10 @@ get_argument(struct archive_string *as, const char *p) /* * Set up command line arguments. - * Returns ARChIVE_OK if everything okey. - * Returns ARChIVE_FAILED if there is a lack of the `"' terminator or an + * Returns ARCHIVE_OK if everything okey. + * Returns ARCHIVE_FAILED if there is a lack of the `"' terminator or an * empty command line. - * Returns ARChIVE_FATAL if no memory. + * Returns ARCHIVE_FATAL if no memory. */ int __archive_cmdline_parse(struct archive_cmdline *data, const char *cmd) diff --git a/contrib/libarchive/libarchive/archive_platform.h b/contrib/libarchive/libarchive/archive_platform.h index 692238277a84..3ade7887222a 100644 --- a/contrib/libarchive/libarchive/archive_platform.h +++ b/contrib/libarchive/libarchive/archive_platform.h @@ -52,6 +52,17 @@ #error Oops: No config.h and no pre-built configuration in archive_platform.h. #endif +/* On macOS check for some symbols based on the deployment target version. */ +#if defined(__APPLE__) +# undef HAVE_FUTIMENS +# undef HAVE_UTIMENSAT +# include <AvailabilityMacros.h> +# if MAC_OS_X_VERSION_MIN_REQUIRED >= 101300 +# define HAVE_FUTIMENS 1 +# define HAVE_UTIMENSAT 1 +# endif +#endif + /* It should be possible to get rid of this by extending the feature-test * macros to cover Windows API functions, probably along with non-trivial * refactoring of code to find structures that sit more cleanly on top of diff --git a/contrib/libarchive/libarchive/archive_read_append_filter.c b/contrib/libarchive/libarchive/archive_read_append_filter.c index 5e4d16307987..da7c55b9b088 100644 --- a/contrib/libarchive/libarchive/archive_read_append_filter.c +++ b/contrib/libarchive/libarchive/archive_read_append_filter.c @@ -89,6 +89,10 @@ archive_read_append_filter(struct archive *_a, int code) strcpy(str, "lz4"); r1 = archive_read_support_filter_lz4(_a); break; + case ARCHIVE_FILTER_ZSTD: + strcpy(str, "zstd"); + r1 = archive_read_support_filter_zstd(_a); + break; case ARCHIVE_FILTER_LZIP: strcpy(str, "lzip"); r1 = archive_read_support_filter_lzip(_a); diff --git a/contrib/libarchive/libarchive/archive_read_disk_entry_from_file.c b/contrib/libarchive/libarchive/archive_read_disk_entry_from_file.c index 548ba89ef3db..1786cff38453 100644 --- a/contrib/libarchive/libarchive/archive_read_disk_entry_from_file.c +++ b/contrib/libarchive/libarchive/archive_read_disk_entry_from_file.c @@ -127,7 +127,7 @@ archive_read_disk_entry_setup_acls(struct archive_read_disk *a, /* * Enter working directory and return working pathname of archive_entry. * If a pointer to an integer is provided and its value is below zero - * open a file descriptor on this pahtname. + * open a file descriptor on this pathname. */ const char * archive_read_disk_entry_setup_path(struct archive_read_disk *a, diff --git a/contrib/libarchive/libarchive/archive_read_filter.3 b/contrib/libarchive/libarchive/archive_read_filter.3 index 7f020e3734b8..ef0a70175332 100644 --- a/contrib/libarchive/libarchive/archive_read_filter.3 +++ b/contrib/libarchive/libarchive/archive_read_filter.3 @@ -38,6 +38,7 @@ .Nm archive_read_support_filter_rpm , .Nm archive_read_support_filter_uu , .Nm archive_read_support_filter_xz , +.Nm archive_read_support_filter_zstd , .Nm archive_read_support_filter_program , .Nm archive_read_support_filter_program_signature .Nd functions for reading streaming archives @@ -73,6 +74,8 @@ Streaming Archive Library (libarchive, -larchive) .Ft int .Fn archive_read_support_filter_xz "struct archive *" .Ft int +.Fn archive_read_support_filter_zstd "struct archive *" +.Ft int .Fo archive_read_support_filter_program .Fa "struct archive *" .Fa "const char *cmd" @@ -99,7 +102,8 @@ Streaming Archive Library (libarchive, -larchive) .Fn archive_read_support_filter_none , .Fn archive_read_support_filter_rpm , .Fn archive_read_support_filter_uu , -.Fn archive_read_support_filter_xz +.Fn archive_read_support_filter_xz , +.Fn archive_read_support_filter_zstd , .Xc Enables auto-detection code and decompression support for the specified compression. diff --git a/contrib/libarchive/libarchive/archive_read_support_filter_all.c b/contrib/libarchive/libarchive/archive_read_support_filter_all.c index 68c53de41f2d..edb508c1dfd8 100644 --- a/contrib/libarchive/libarchive/archive_read_support_filter_all.c +++ b/contrib/libarchive/libarchive/archive_read_support_filter_all.c @@ -71,6 +71,8 @@ archive_read_support_filter_all(struct archive *a) archive_read_support_filter_grzip(a); /* Lz4 falls back to "lz4 -d" command-line program. */ archive_read_support_filter_lz4(a); + /* Zstd falls back to "zstd -d" command-line program. */ + archive_read_support_filter_zstd(a); /* Note: We always return ARCHIVE_OK here, even if some of the * above return ARCHIVE_WARN. The intent here is to enable diff --git a/contrib/libarchive/libarchive/archive_read_support_filter_zstd.c b/contrib/libarchive/libarchive/archive_read_support_filter_zstd.c new file mode 100644 index 000000000000..c8bb36be703c --- /dev/null +++ b/contrib/libarchive/libarchive/archive_read_support_filter_zstd.c @@ -0,0 +1,292 @@ +/*- + * Copyright (c) 2009-2011 Sean Purcell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "archive_platform.h" + +__FBSDID("$FreeBSD$"); + +#ifdef HAVE_ERRNO_H +#include <errno.h> +#endif + +#ifdef HAVE_ERRNO_H +#include <errno.h> +#endif +#include <stdio.h> +#ifdef HAVE_STDLIB_H +#include <stdlib.h> +#endif +#ifdef HAVE_STRING_H +#include <string.h> +#endif +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif +#if HAVE_ZSTD_H +#include <zstd.h> +#endif + +#include "archive.h" +#include "archive_endian.h" +#include "archive_private.h" +#include "archive_read_private.h" + +#if HAVE_ZSTD_H && HAVE_LIBZSTD + +struct private_data { + ZSTD_DStream *dstream; + unsigned char *out_block; + size_t out_block_size; + int64_t total_out; + char in_frame; /* True = in the middle of a zstd frame. */ + char eof; /* True = found end of compressed data. */ +}; + +/* Zstd Filter. */ +static ssize_t zstd_filter_read(struct archive_read_filter *, const void**); +static int zstd_filter_close(struct archive_read_filter *); +#endif + +/* + * Note that we can detect zstd compressed files even if we can't decompress + * them. (In fact, we like detecting them because we can give better error + * messages.) So the bid framework here gets compiled even if no zstd library + * is available. + */ +static int zstd_bidder_bid(struct archive_read_filter_bidder *, + struct archive_read_filter *); +static int zstd_bidder_init(struct archive_read_filter *); + +int +archive_read_support_filter_zstd(struct archive *_a) +{ + struct archive_read *a = (struct archive_read *)_a; + struct archive_read_filter_bidder *bidder; + + archive_check_magic(_a, ARCHIVE_READ_MAGIC, + ARCHIVE_STATE_NEW, "archive_read_support_filter_zstd"); + + if (__archive_read_get_bidder(a, &bidder) != ARCHIVE_OK) + return (ARCHIVE_FATAL); + + bidder->data = NULL; + bidder->name = "zstd"; + bidder->bid = zstd_bidder_bid; + bidder->init = zstd_bidder_init; + bidder->options = NULL; + bidder->free = NULL; +#if HAVE_ZSTD_H && HAVE_LIBZSTD + return (ARCHIVE_OK); +#else + archive_set_error(_a, ARCHIVE_ERRNO_MISC, + "Using external zstd program for zstd decompression"); + return (ARCHIVE_WARN); +#endif +} + +/* + * Test whether we can handle this data. + */ +static int +zstd_bidder_bid(struct archive_read_filter_bidder *self, + struct archive_read_filter *filter) +{ + const unsigned char *buffer; + ssize_t avail; + unsigned prefix; + + /* Zstd frame magic values */ + const unsigned zstd_magic = 0xFD2FB528U; + + (void) self; /* UNUSED */ + + buffer = __archive_read_filter_ahead(filter, 4, &avail); + if (buffer == NULL) + return (0); + + prefix = archive_le32dec(buffer); + if (prefix == zstd_magic) + return (32); + + return (0); +} + +#if !(HAVE_ZSTD_H && HAVE_LIBZSTD) + +/* + * If we don't have the library on this system, we can't do the + * decompression directly. We can, however, try to run "zstd -d" + * in case that's available. + */ +static int +zstd_bidder_init(struct archive_read_filter *self) +{ + int r; + + r = __archive_read_program(self, "zstd -d -qq"); + /* Note: We set the format here even if __archive_read_program() + * above fails. We do, after all, know what the format is + * even if we weren't able to read it. */ + self->code = ARCHIVE_FILTER_ZSTD; + self->name = "zstd"; + return (r); +} + +#else + +/* + * Initialize the filter object + */ +static int +zstd_bidder_init(struct archive_read_filter *self) +{ + struct private_data *state; + const size_t out_block_size = ZSTD_DStreamOutSize(); + void *out_block; + ZSTD_DStream *dstream; + + self->code = ARCHIVE_FILTER_ZSTD; + self->name = "zstd"; + + state = (struct private_data *)calloc(sizeof(*state), 1); + out_block = (unsigned char *)malloc(out_block_size); + dstream = ZSTD_createDStream(); + + if (state == NULL || out_block == NULL || dstream == NULL) { + free(out_block); + free(state); + ZSTD_freeDStream(dstream); /* supports free on NULL */ + archive_set_error(&self->archive->archive, ENOMEM, + "Can't allocate data for zstd decompression"); + return (ARCHIVE_FATAL); + } + + self->data = state; + + state->out_block_size = out_block_size; + state->out_block = out_block; + state->dstream = dstream; + self->read = zstd_filter_read; + self->skip = NULL; /* not supported */ + self->close = zstd_filter_close; + + state->eof = 0; + state->in_frame = 0; + + return (ARCHIVE_OK); +} + +static ssize_t +zstd_filter_read(struct archive_read_filter *self, const void **p) +{ + struct private_data *state; + size_t decompressed; + ssize_t avail_in; + ZSTD_outBuffer out; + ZSTD_inBuffer in; + + state = (struct private_data *)self->data; + + out = (ZSTD_outBuffer) { state->out_block, state->out_block_size, 0 }; + + /* Try to fill the output buffer. */ + while (out.pos < out.size && !state->eof) { + if (!state->in_frame) { + const size_t ret = ZSTD_initDStream(state->dstream); + if (ZSTD_isError(ret)) { + archive_set_error(&self->archive->archive, + ARCHIVE_ERRNO_MISC, + "Error initializing zstd decompressor: %s", + ZSTD_getErrorName(ret)); + return (ARCHIVE_FATAL); + } + } + in.src = __archive_read_filter_ahead(self->upstream, 1, + &avail_in); + if (avail_in < 0) { + return avail_in; + } + if (in.src == NULL && avail_in == 0) { + if (!state->in_frame) { + /* end of stream */ + state->eof = 1; + break; + } else { + archive_set_error(&self->archive->archive, + ARCHIVE_ERRNO_MISC, + "Truncated zstd input"); + return (ARCHIVE_FATAL); + } + } + in.size = avail_in; + in.pos = 0; + + { + const size_t ret = + ZSTD_decompressStream(state->dstream, &out, &in); + + if (ZSTD_isError(ret)) { + archive_set_error(&self->archive->archive, + ARCHIVE_ERRNO_MISC, + "Zstd decompression failed: %s", + ZSTD_getErrorName(ret)); + return (ARCHIVE_FATAL); + } + + /* Decompressor made some progress */ + __archive_read_filter_consume(self->upstream, in.pos); + + /* ret guaranteed to be > 0 if frame isn't done yet */ + state->in_frame = (ret != 0); + } + } + + decompressed = out.pos; + state->total_out += decompressed; + if (decompressed == 0) + *p = NULL; + else + *p = state->out_block; + return (decompressed); +} + +/* + * Clean up the decompressor. + */ +static int +zstd_filter_close(struct archive_read_filter *self) +{ + struct private_data *state; + + state = (struct private_data *)self->data; + + ZSTD_freeDStream(state->dstream); + free(state->out_block); + free(state); + + return (ARCHIVE_OK); +} + +#endif /* HAVE_ZLIB_H && HAVE_LIBZSTD */ diff --git a/contrib/libarchive/libarchive/archive_read_support_format_cpio.c b/contrib/libarchive/libarchive/archive_read_support_format_cpio.c index 6b4d241fd34a..665acaeff459 100644 --- a/contrib/libarchive/libarchive/archive_read_support_format_cpio.c +++ b/contrib/libarchive/libarchive/archive_read_support_format_cpio.c @@ -633,6 +633,13 @@ header_newc(struct archive_read *a, struct cpio *cpio, /* Pad name to 2 more than a multiple of 4. */ *name_pad = (2 - *namelength) & 3; + /* Make sure that the padded name length fits into size_t. */ + if (*name_pad > SIZE_MAX - *namelength) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, + "cpio archive has invalid namelength"); + return (ARCHIVE_FATAL); + } + /* * Note: entry_bytes_remaining is at least 64 bits and * therefore guaranteed to be big enough for a 33-bit file diff --git a/contrib/libarchive/libarchive/archive_read_support_format_mtree.c b/contrib/libarchive/libarchive/archive_read_support_format_mtree.c index d2770f1cf701..69a62e189bc5 100644 --- a/contrib/libarchive/libarchive/archive_read_support_format_mtree.c +++ b/contrib/libarchive/libarchive/archive_read_support_format_mtree.c @@ -77,6 +77,8 @@ __FBSDID("$FreeBSD$"); #define MTREE_HASHTABLE_SIZE 1024 +#define MAX_LINE_LEN (1024 * 1024) + struct mtree_option { struct mtree_option *next; char *value; @@ -334,6 +336,14 @@ next_line(struct archive_read *a, size_t nbytes_req = (*ravail+1023) & ~1023U; ssize_t tested; + /* + * Place an arbitrary limit on the line length. + * mtree is almost free-form input and without line length limits, + * it can consume a lot of memory. + */ + if (len >= MAX_LINE_LEN) + return (-1); + /* Increase reading bytes if it is not enough to at least * new two lines. */ if (nbytes_req < (size_t)*ravail + 160) diff --git a/contrib/libarchive/libarchive/archive_read_support_format_rar.c b/contrib/libarchive/libarchive/archive_read_support_format_rar.c index cbb14c32dc3b..751de6979ba5 100644 --- a/contrib/libarchive/libarchive/archive_read_support_format_rar.c +++ b/contrib/libarchive/libarchive/archive_read_support_format_rar.c @@ -1496,7 +1496,11 @@ read_header(struct archive_read *a, struct archive_entry *entry, return (ARCHIVE_FATAL); } filename[filename_size++] = '\0'; - filename[filename_size++] = '\0'; + /* + * Do not increment filename_size here as the computations below + * add the space for the terminating NUL explicitly. + */ + filename[filename_size] = '\0'; /* Decoded unicode form is UTF-16BE, so we have to update a string * conversion object for it. */ diff --git a/contrib/libarchive/libarchive/archive_read_support_format_tar.c b/contrib/libarchive/libarchive/archive_read_support_format_tar.c index e956f40cd6ad..7348e2d5c07e 100644 --- a/contrib/libarchive/libarchive/archive_read_support_format_tar.c +++ b/contrib/libarchive/libarchive/archive_read_support_format_tar.c @@ -2243,7 +2243,7 @@ gnu_add_sparse_entry(struct archive_read *a, struct tar *tar, else tar->sparse_list = p; tar->sparse_last = p; - if (remaining < 0 || offset < 0) { + if (remaining < 0 || offset < 0 || offset > INT64_MAX - remaining) { archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, "Malformed sparse map data"); return (ARCHIVE_FATAL); } diff --git a/contrib/libarchive/libarchive/archive_read_support_format_xar.c b/contrib/libarchive/libarchive/archive_read_support_format_xar.c index 7a22beb9d8e4..602fc7722144 100644 --- a/contrib/libarchive/libarchive/archive_read_support_format_xar.c +++ b/contrib/libarchive/libarchive/archive_read_support_format_xar.c @@ -1040,6 +1040,9 @@ atol10(const char *p, size_t char_cnt) uint64_t l; int digit; + if (char_cnt == 0) + return (0); + l = 0; digit = *p - '0'; while (digit >= 0 && digit < 10 && char_cnt-- > 0) { @@ -1054,7 +1057,10 @@ atol8(const char *p, size_t char_cnt) { int64_t l; int digit; - + + if (char_cnt == 0) + return (0); + l = 0; while (char_cnt-- > 0) { if (*p >= '0' && *p <= '7') @@ -2623,6 +2629,14 @@ strappend_base64(struct xar *xar, archive_strncat(as, (const char *)buff, len); } +static int +is_string(const char *known, const char *data, size_t len) +{ + if (strlen(known) != len) + return -1; + return memcmp(data, known, len); +} + static void xml_data(void *userData, const char *s, int len) { @@ -2674,26 +2688,26 @@ xml_data(void *userData, const char *s, int len) archive_strncpy(&(xar->file->symlink), s, len); break; case FILE_TYPE: - if (strncmp("file", s, len) == 0 || - strncmp("hardlink", s, len) == 0) + if (is_string("file", s, len) == 0 || + is_string("hardlink", s, len) == 0) xar->file->mode = (xar->file->mode & ~AE_IFMT) | AE_IFREG; - if (strncmp("directory", s, len) == 0) + if (is_string("directory", s, len) == 0) xar->file->mode = (xar->file->mode & ~AE_IFMT) | AE_IFDIR; - if (strncmp("symlink", s, len) == 0) + if (is_string("symlink", s, len) == 0) xar->file->mode = (xar->file->mode & ~AE_IFMT) | AE_IFLNK; - if (strncmp("character special", s, len) == 0) + if (is_string("character special", s, len) == 0) xar->file->mode = (xar->file->mode & ~AE_IFMT) | AE_IFCHR; - if (strncmp("block special", s, len) == 0) + if (is_string("block special", s, len) == 0) xar->file->mode = (xar->file->mode & ~AE_IFMT) | AE_IFBLK; - if (strncmp("socket", s, len) == 0) + if (is_string("socket", s, len) == 0) xar->file->mode = (xar->file->mode & ~AE_IFMT) | AE_IFSOCK; - if (strncmp("fifo", s, len) == 0) + if (is_string("fifo", s, len) == 0) xar->file->mode = (xar->file->mode & ~AE_IFMT) | AE_IFIFO; xar->file->has |= HAS_TYPE; diff --git a/contrib/libarchive/libarchive/archive_read_support_format_zip.c b/contrib/libarchive/libarchive/archive_read_support_format_zip.c index c88e7bbbe0d5..d6c957045625 100644 --- a/contrib/libarchive/libarchive/archive_read_support_format_zip.c +++ b/contrib/libarchive/libarchive/archive_read_support_format_zip.c @@ -723,6 +723,11 @@ process_extra(struct archive_read *a, const char *p, size_t extra_length, struct } case 0x9901: /* WinZip AES extra data field. */ + if (datasize < 6) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, + "Incomplete AES field"); + return ARCHIVE_FAILED; + } if (p[offset + 2] == 'A' && p[offset + 3] == 'E') { /* Vendor version. */ zip_entry->aes_extra.vendor = diff --git a/contrib/libarchive/libarchive/archive_string.c b/contrib/libarchive/libarchive/archive_string.c index 3501a9e09143..c12355f08916 100644 --- a/contrib/libarchive/libarchive/archive_string.c +++ b/contrib/libarchive/libarchive/archive_string.c @@ -214,7 +214,8 @@ archive_wstring_append(struct archive_wstring *as, const wchar_t *p, size_t s) { if (archive_wstring_ensure(as, as->length + s + 1) == NULL) return (NULL); - wmemmove(as->s + as->length, p, s); + if (s) + wmemmove(as->s + as->length, p, s); as->length += s; as->s[as->length] = 0; return (as); diff --git a/contrib/libarchive/libarchive/archive_version_details.c b/contrib/libarchive/libarchive/archive_version_details.c index e10b83e5b7e4..2651503e6fda 100644 --- a/contrib/libarchive/libarchive/archive_version_details.c +++ b/contrib/libarchive/libarchive/archive_version_details.c @@ -45,6 +45,9 @@ __FBSDID("$FreeBSD$"); #ifdef HAVE_LZ4_H #include <lz4.h> #endif +#ifdef HAVE_ZSTD_H +#include <zstd.h> +#endif #include "archive.h" #include "archive_private.h" @@ -59,6 +62,7 @@ archive_version_details(void) const char *liblzma = archive_liblzma_version(); const char *bzlib = archive_bzlib_version(); const char *liblz4 = archive_liblz4_version(); + const char *libzstd = archive_libzstd_version(); if (!init) { archive_string_init(&str); @@ -84,6 +88,10 @@ archive_version_details(void) archive_strcat(&str, " liblz4/"); archive_strcat(&str, liblz4); } + if (libzstd) { + archive_strcat(&str, " libzstd/"); + archive_strcat(&str, libzstd); + } } return str.s; } @@ -131,3 +139,13 @@ archive_liblz4_version(void) return NULL; #endif } + +const char * +archive_libzstd_version(void) +{ +#if HAVE_ZSTD_H && HAVE_LIBZSTD + return ZSTD_VERSION_STRING; +#else + return NULL; +#endif +} diff --git a/contrib/libarchive/libarchive/archive_write.3 b/contrib/libarchive/libarchive/archive_write.3 index 376d71dee20e..c1164f5b5fdb 100644 --- a/contrib/libarchive/libarchive/archive_write.3 +++ b/contrib/libarchive/libarchive/archive_write.3 @@ -71,7 +71,7 @@ support. .\" .Ss Set options See -.Xr archive_read_set_options 3 . +.Xr archive_write_set_options 3 . .\" .Ss Open archive See diff --git a/contrib/libarchive/libarchive/archive_write_add_filter.c b/contrib/libarchive/libarchive/archive_write_add_filter.c index 08f518adec4b..203f4142b5c9 100644 --- a/contrib/libarchive/libarchive/archive_write_add_filter.c +++ b/contrib/libarchive/libarchive/archive_write_add_filter.c @@ -53,6 +53,7 @@ struct { int code; int (*setter)(struct archive *); } codes[] = { ARCHIVE_FILTER_LZOP, archive_write_add_filter_lzip }, { ARCHIVE_FILTER_UU, archive_write_add_filter_uuencode }, { ARCHIVE_FILTER_XZ, archive_write_add_filter_xz }, + { ARCHIVE_FILTER_ZSTD, archive_write_add_filter_zstd }, { -1, NULL } }; diff --git a/contrib/libarchive/libarchive/archive_write_add_filter_by_name.c b/contrib/libarchive/libarchive/archive_write_add_filter_by_name.c index 85a8d4753415..ffa633c96371 100644 --- a/contrib/libarchive/libarchive/archive_write_add_filter_by_name.c +++ b/contrib/libarchive/libarchive/archive_write_add_filter_by_name.c @@ -57,6 +57,7 @@ struct { const char *name; int (*setter)(struct archive *); } names[] = { "lzop", archive_write_add_filter_lzop }, { "uuencode", archive_write_add_filter_uuencode }, { "xz", archive_write_add_filter_xz }, + { "zstd", archive_write_add_filter_zstd }, { NULL, NULL } }; diff --git a/contrib/libarchive/libarchive/archive_write_add_filter_zstd.c b/contrib/libarchive/libarchive/archive_write_add_filter_zstd.c new file mode 100644 index 000000000000..671fc6affbaa --- /dev/null +++ b/contrib/libarchive/libarchive/archive_write_add_filter_zstd.c @@ -0,0 +1,335 @@ +/*- + * Copyright (c) 2017 Sean Purcell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "archive_platform.h" + +__FBSDID("$FreeBSD$"); + + +#ifdef HAVE_ERRNO_H +#include <errno.h> +#endif +#ifdef HAVE_STDLIB_H +#include <stdlib.h> +#endif +#ifdef HAVE_STRING_H +#include <string.h> +#endif +#ifdef HAVE_ZSTD_H +#include <zstd.h> +#endif + +#include "archive.h" +#include "archive_private.h" +#include "archive_string.h" +#include "archive_write_private.h" + +/* Don't compile this if we don't have zstd.h */ + +struct private_data { + int compression_level; +#if HAVE_ZSTD_H && HAVE_LIBZSTD + ZSTD_CStream *cstream; + int64_t total_in; + ZSTD_outBuffer out; +#else + struct archive_write_program_data *pdata; +#endif +}; + +static int archive_compressor_zstd_options(struct archive_write_filter *, + const char *, const char *); +static int archive_compressor_zstd_open(struct archive_write_filter *); +static int archive_compressor_zstd_write(struct archive_write_filter *, + const void *, size_t); +static int archive_compressor_zstd_close(struct archive_write_filter *); +static int archive_compressor_zstd_free(struct archive_write_filter *); +#if HAVE_ZSTD_H && HAVE_LIBZSTD +static int drive_compressor(struct archive_write_filter *, + struct private_data *, int, const void *, size_t); +#endif + + +/* + * Add a zstd compression filter to this write handle. + */ +int +archive_write_add_filter_zstd(struct archive *_a) +{ + struct archive_write *a = (struct archive_write *)_a; + struct archive_write_filter *f = __archive_write_allocate_filter(_a); + struct private_data *data; + archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC, + ARCHIVE_STATE_NEW, "archive_write_add_filter_zstd"); + + data = calloc(1, sizeof(*data)); + if (data == NULL) { + archive_set_error(&a->archive, ENOMEM, "Out of memory"); + return (ARCHIVE_FATAL); + } + f->data = data; + f->open = &archive_compressor_zstd_open; + f->options = &archive_compressor_zstd_options; + f->close = &archive_compressor_zstd_close; + f->free = &archive_compressor_zstd_free; + f->code = ARCHIVE_FILTER_ZSTD; + f->name = "zstd"; + data->compression_level = 3; /* Default level used by the zstd CLI */ +#if HAVE_ZSTD_H && HAVE_LIBZSTD + data->cstream = ZSTD_createCStream(); + if (data->cstream == NULL) { + free(data); + archive_set_error(&a->archive, ENOMEM, + "Failed to allocate zstd compressor object"); + return (ARCHIVE_FATAL); + } + + return (ARCHIVE_OK); +#else + data->pdata = __archive_write_program_allocate("zstd"); + if (data->pdata == NULL) { + free(data); + archive_set_error(&a->archive, ENOMEM, "Out of memory"); + return (ARCHIVE_FATAL); + } + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Using external zstd program"); + return (ARCHIVE_WARN); +#endif +} + +static int +archive_compressor_zstd_free(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; +#if HAVE_ZSTD_H && HAVE_LIBZSTD + ZSTD_freeCStream(data->cstream); + free(data->out.dst); +#else + __archive_write_program_free(data->pdata); +#endif + free(data); + f->data = NULL; + return (ARCHIVE_OK); +} + +/* + * Set write options. + */ +static int +archive_compressor_zstd_options(struct archive_write_filter *f, const char *key, + const char *value) +{ + struct private_data *data = (struct private_data *)f->data; + + if (strcmp(key, "compression-level") == 0) { + int level = atoi(value); +#if HAVE_ZSTD_H && HAVE_LIBZSTD + if (level < 1 || level > ZSTD_maxCLevel()) { +#else + /* If we don't have the library, hard-code the max level */ + if (level < 1 || level > 22) { +#endif + return (ARCHIVE_WARN); + } + data->compression_level = level; + return (ARCHIVE_OK); + } + + /* Note: The "warn" return is just to inform the options + * supervisor that we didn't handle it. It will generate + * a suitable error if no one used this option. */ + return (ARCHIVE_WARN); +} + +#if HAVE_ZSTD_H && HAVE_LIBZSTD +/* + * Setup callback. + */ +static int +archive_compressor_zstd_open(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; + int ret; + + ret = __archive_write_open_filter(f->next_filter); + if (ret != ARCHIVE_OK) + return (ret); + + if (data->out.dst == NULL) { + size_t bs = ZSTD_CStreamOutSize(), bpb; + if (f->archive->magic == ARCHIVE_WRITE_MAGIC) { + /* Buffer size should be a multiple number of + * the of bytes per block for performance. */ + bpb = archive_write_get_bytes_per_block(f->archive); + if (bpb > bs) + bs = bpb; + else if (bpb != 0) + bs -= bs % bpb; + } + data->out.size = bs; + data->out.pos = 0; + data->out.dst + = (unsigned char *)malloc(data->out.size); + if (data->out.dst == NULL) { + archive_set_error(f->archive, ENOMEM, + "Can't allocate data for compression buffer"); + return (ARCHIVE_FATAL); + } + } + + f->write = archive_compressor_zstd_write; + + if (ZSTD_isError(ZSTD_initCStream(data->cstream, + data->compression_level))) { + archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, + "Internal error initializing zstd compressor object"); + return (ARCHIVE_FATAL); + } + + return (ARCHIVE_OK); +} + +/* + * Write data to the compressed stream. + */ +static int +archive_compressor_zstd_write(struct archive_write_filter *f, const void *buff, + size_t length) +{ + struct private_data *data = (struct private_data *)f->data; + int ret; + + /* Update statistics */ + data->total_in += length; + + if ((ret = drive_compressor(f, data, 0, buff, length)) != ARCHIVE_OK) + return (ret); + + return (ARCHIVE_OK); +} + +/* + * Finish the compression... + */ +static int +archive_compressor_zstd_close(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; + int r1, r2; + + /* Finish zstd frame */ + r1 = drive_compressor(f, data, 1, NULL, 0); + + r2 = __archive_write_close_filter(f->next_filter); + + return r1 < r2 ? r1 : r2; +} + +/* + * Utility function to push input data through compressor, + * writing full output blocks as necessary. + * + * Note that this handles both the regular write case (finishing == + * false) and the end-of-archive case (finishing == true). + */ +static int +drive_compressor(struct archive_write_filter *f, + struct private_data *data, int finishing, const void *src, size_t length) +{ + ZSTD_inBuffer in = (ZSTD_inBuffer) { src, length, 0 }; + + for (;;) { + if (data->out.pos == data->out.size) { + const int ret = __archive_write_filter(f->next_filter, + data->out.dst, data->out.size); + if (ret != ARCHIVE_OK) + return (ARCHIVE_FATAL); + data->out.pos = 0; + } + + /* If there's nothing to do, we're done. */ + if (!finishing && in.pos == in.size) + return (ARCHIVE_OK); + + { + const size_t zstdret = !finishing ? + ZSTD_compressStream(data->cstream, &data->out, &in) + : ZSTD_endStream(data->cstream, &data->out); + + if (ZSTD_isError(zstdret)) { + archive_set_error(f->archive, + ARCHIVE_ERRNO_MISC, + "Zstd compression failed: %s", + ZSTD_getErrorName(zstdret)); + return (ARCHIVE_FATAL); + } + + /* If we're finishing, 0 means nothing left to flush */ + if (finishing && zstdret == 0) { + const int ret = __archive_write_filter(f->next_filter, + data->out.dst, data->out.pos); + return (ret); + } + } + } +} + +#else /* HAVE_ZSTD_H && HAVE_LIBZSTD */ + +static int +archive_compressor_zstd_open(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; + struct archive_string as; + int r; + + archive_string_init(&as); + archive_string_sprintf(&as, "zstd -%d", data->compression_level); + + f->write = archive_compressor_zstd_write; + r = __archive_write_program_open(f, data->pdata, as.s); + archive_string_free(&as); + return (r); +} + +static int +archive_compressor_zstd_write(struct archive_write_filter *f, const void *buff, + size_t length) +{ + struct private_data *data = (struct private_data *)f->data; + + return __archive_write_program_write(f, data->pdata, buff, length); +} + +static int +archive_compressor_zstd_close(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; + + return __archive_write_program_close(f, data->pdata); +} + +#endif /* HAVE_ZSTD_H && HAVE_LIBZSTD */ diff --git a/contrib/libarchive/libarchive/archive_write_filter.3 b/contrib/libarchive/libarchive/archive_write_filter.3 index e1d189150644..d6fa07131a80 100644 --- a/contrib/libarchive/libarchive/archive_write_filter.3 +++ b/contrib/libarchive/libarchive/archive_write_filter.3 @@ -42,7 +42,8 @@ .Nm archive_write_add_filter_none , .Nm archive_write_add_filter_program , .Nm archive_write_add_filter_uuencode , -.Nm archive_write_add_filter_xz +.Nm archive_write_add_filter_xz , +.Nm archive_write_add_filter_zstd , .Nd functions enabling output filters .Sh LIBRARY Streaming Archive Library (libarchive, -larchive) @@ -76,6 +77,8 @@ Streaming Archive Library (libarchive, -larchive) .Fn archive_write_add_filter_uuencode "struct archive *" .Ft int .Fn archive_write_add_filter_xz "struct archive *" +.Ft int +.Fn archive_write_add_filter_zstd "struct archive *" .Sh DESCRIPTION .Bl -tag -width indent .It Xo @@ -89,6 +92,7 @@ Streaming Archive Library (libarchive, -larchive) .Fn archive_write_add_filter_lzma , .Fn archive_write_add_filter_lzop , .Fn archive_write_add_filter_xz , +.Fn archive_write_add_filter_zstd , .Xc The resulting archive will be compressed as specified. Note that the compressed output is always properly blocked. diff --git a/contrib/libarchive/libarchive/archive_write_set_format_pax.c b/contrib/libarchive/libarchive/archive_write_set_format_pax.c index 318ca78c554d..650b4529060a 100644 --- a/contrib/libarchive/libarchive/archive_write_set_format_pax.c +++ b/contrib/libarchive/libarchive/archive_write_set_format_pax.c @@ -1654,7 +1654,7 @@ build_pax_attribute_name(char *dest, const char *src) * GNU PAX Format 1.0 requires the special name, which pattern is: * <dir>/GNUSparseFile.<pid>/<original file name> * - * Since reproducable archives are more important, use 0 as pid. + * Since reproducible archives are more important, use 0 as pid. * * This function is used for only Sparse file, a file type of which * is regular file. diff --git a/contrib/libarchive/libarchive/test/test_archive_write_add_filter_by_name.c b/contrib/libarchive/libarchive/test/test_archive_write_add_filter_by_name.c index 38b606b9d165..d962af92311b 100644 --- a/contrib/libarchive/libarchive/test/test_archive_write_add_filter_by_name.c +++ b/contrib/libarchive/libarchive/test/test_archive_write_add_filter_by_name.c @@ -207,3 +207,8 @@ DEFINE_TEST(test_archive_write_add_filter_by_name_xz) { test_filter_by_name("xz", ARCHIVE_FILTER_XZ, cannot); } + +DEFINE_TEST(test_archive_write_add_filter_by_name_zstd) +{ + test_filter_by_name("zstd", ARCHIVE_FILTER_ZSTD, canZstd); +} diff --git a/contrib/libarchive/libarchive/test/test_compat_zstd.c b/contrib/libarchive/libarchive/test/test_compat_zstd.c new file mode 100644 index 000000000000..1d63a2845ff5 --- /dev/null +++ b/contrib/libarchive/libarchive/test/test_compat_zstd.c @@ -0,0 +1,82 @@ +/*- + * Copyright (c) 2017 Sean Purcell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "test.h" +__FBSDID("$FreeBSD$"); + +/* + * Verify our ability to read sample files compatibly with 'zstd -d'. + * + * In particular: + * * zstd -d will read multiple zstd streams, concatenating the output + * * zstd -d will skip over zstd skippable frames + */ + +static void +compat_zstd(const char *name) +{ + const char *n[7] = { "f1", "f2", "f3", "d1/f1", "d1/f2", "d1/f3", NULL }; + struct archive_entry *ae; + struct archive *a; + int i, r; + + assert((a = archive_read_new()) != NULL); + assertEqualIntA(a, ARCHIVE_OK, archive_read_support_filter_all(a)); + r = archive_read_support_filter_zstd(a); + if (r == ARCHIVE_WARN) { + skipping("zstd reading not fully supported on this platform"); + assertEqualInt(ARCHIVE_OK, archive_read_free(a)); + return; + } + assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_all(a)); + extract_reference_file(name); + assertEqualIntA(a, ARCHIVE_OK, archive_read_open_filename(a, name, 2)); + + /* Read entries, match up names with list above. */ + for (i = 0; i < 6; ++i) { + failure("Could not read file %d (%s) from %s", i, n[i], name); + assertEqualIntA(a, ARCHIVE_OK, + archive_read_next_header(a, &ae)); + assertEqualString(n[i], archive_entry_pathname(ae)); + } + + /* Verify the end-of-archive. */ + assertEqualIntA(a, ARCHIVE_EOF, archive_read_next_header(a, &ae)); + + /* Verify that the format detection worked. */ + assertEqualInt(archive_filter_code(a, 0), ARCHIVE_FILTER_ZSTD); + assertEqualString(archive_filter_name(a, 0), "zstd"); + assertEqualInt(archive_format(a), ARCHIVE_FORMAT_TAR_USTAR); + + assertEqualInt(ARCHIVE_OK, archive_read_close(a)); + assertEqualInt(ARCHIVE_OK, archive_read_free(a)); +} + + +DEFINE_TEST(test_compat_zstd) +{ + /* This sample was compressed as 3 separate streams with a zstd skippable + * frame placed in the middle */ + compat_zstd("test_compat_zstd_1.tar.zst"); +} diff --git a/contrib/libarchive/libarchive/test/test_compat_zstd_1.tar.zst.uu b/contrib/libarchive/libarchive/test/test_compat_zstd_1.tar.zst.uu new file mode 100644 index 000000000000..4b0b98447ffa --- /dev/null +++ b/contrib/libarchive/libarchive/test/test_compat_zstd_1.tar.zst.uu @@ -0,0 +1,12 @@ +begin 644 test_compat_zstd_1.tar.zst +M*+4O_010)0,`HL0.%;`Q&>>\/$2[#IQF[<1+Z3T<0CX]!77&0@R.6+/F,0+I +M.$1A$QE2`J!+*_6[_YT9_W_M1KC-EG*V>10.`,M`%3*@#F#\`-FT#J:1#U1" +M`H1!&R#<!.<"@#3@M58XY1,8`DMMD\@HM2_]!%!=`P`B!1`5H#D!0!.SELJ" +M5#509I*T/YQ^]?H/3T1D>A5\*'"JYIJ;C&4=B2CL(L)*E-IJT/RV?.:A_]_N +MB&[7SDG;/=4&#P";0!5D0`=8T0&R&19,)1^HA`0(@S9`N`G.!0!IP&NM<,K! +M-#8!%A]U]K10*DT8!`````$"`P0HM2_]!%!]`P`B11`6H+$)"%]@,Z6OH`"L +MM$R2MAN&*MSG`W?OJ7+4P*B::VXR`NM(1&$7&58"J*U'_&V^S$/_O]U1N%T[ +M)VW7J'+4!A``_4$%^T`],J`8P.0!L@D63"4?J(0$"(,V0+@)S@4`:<!KK7!J +)P51V`E@!9CD# +` +end diff --git a/contrib/libarchive/libarchive/test/test_fuzz.c b/contrib/libarchive/libarchive/test/test_fuzz.c index 61c2dfc0fcdf..76557d541d00 100644 --- a/contrib/libarchive/libarchive/test/test_fuzz.c +++ b/contrib/libarchive/libarchive/test/test_fuzz.c @@ -413,6 +413,12 @@ DEFINE_TEST(test_fuzz_tar) NULL }; #endif +#if HAVE_ZSTD_H && HAVE_LIBZSTD + static const char *fileset10[] = { + "test_compat_zstd_1.tar.zst", + NULL + }; +#endif static const struct files filesets[] = { {0, fileset1}, /* Exercise bzip2 decompressor. */ {1, fileset1}, @@ -426,6 +432,9 @@ DEFINE_TEST(test_fuzz_tar) #if HAVE_LIBLZO2 && HAVE_LZO_LZO1X_H && HAVE_LZO_LZOCONF_H {0, fileset9}, /* Exercise lzo decompressor. */ #endif +#if HAVE_ZSTD_H && HAVE_LIBZSTD + {0, fileset10}, /* Excercise zstd decompressor. */ +#endif {1, NULL} }; test_fuzz(filesets); diff --git a/contrib/libarchive/libarchive/test/test_write_filter_zstd.c b/contrib/libarchive/libarchive/test/test_write_filter_zstd.c new file mode 100644 index 000000000000..5637aa5e29b3 --- /dev/null +++ b/contrib/libarchive/libarchive/test/test_write_filter_zstd.c @@ -0,0 +1,201 @@ +/*- + * Copyright (c) 2017 Sean Purcell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer + * in this position and unchanged. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" +__FBSDID("$FreeBSD$"); + +DEFINE_TEST(test_write_filter_zstd) +{ + struct archive_entry *ae; + struct archive *a; + char *buff, *data; + size_t buffsize, datasize; + char path[16]; + size_t used1, used2; + int i, r; + + buffsize = 2000000; + assert(NULL != (buff = (char *)malloc(buffsize))); + if (buff == NULL) + return; + + datasize = 10000; + assert(NULL != (data = (char *)malloc(datasize))); + if (data == NULL) { + free(buff); + return; + } + memset(data, 0, datasize); + + /* + * Write a 100 files and read them all back. + */ + assert((a = archive_write_new()) != NULL); + assertEqualIntA(a, ARCHIVE_OK, archive_write_set_format_ustar(a)); + r = archive_write_add_filter_zstd(a); + if (r != ARCHIVE_OK) { + skipping("zstd writing not supported on this platform"); + assertEqualInt(ARCHIVE_OK, archive_write_free(a)); + free(buff); + free(data); + return; + } + assertEqualIntA(a, ARCHIVE_OK, + archive_write_set_bytes_per_block(a, 10)); + assertEqualInt(ARCHIVE_FILTER_ZSTD, archive_filter_code(a, 0)); + assertEqualString("zstd", archive_filter_name(a, 0)); + assertEqualIntA(a, ARCHIVE_OK, archive_write_open_memory(a, buff, buffsize, &used1)); + assertEqualInt(ARCHIVE_FILTER_ZSTD, archive_filter_code(a, 0)); + assertEqualString("zstd", archive_filter_name(a, 0)); + assert((ae = archive_entry_new()) != NULL); + archive_entry_set_filetype(ae, AE_IFREG); + archive_entry_set_size(ae, datasize); + for (i = 0; i < 100; i++) { + sprintf(path, "file%03d", i); + archive_entry_copy_pathname(ae, path); + assertEqualIntA(a, ARCHIVE_OK, archive_write_header(a, ae)); + assertA(datasize + == (size_t)archive_write_data(a, data, datasize)); + } + archive_entry_free(ae); + assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a)); + assertEqualInt(ARCHIVE_OK, archive_write_free(a)); + + assert((a = archive_read_new()) != NULL); + assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_all(a)); + r = archive_read_support_filter_zstd(a); + if (r == ARCHIVE_WARN) { + skipping("Can't verify zstd writing by reading back;" + " zstd reading not fully supported on this platform"); + } else { + assertEqualIntA(a, ARCHIVE_OK, + archive_read_support_filter_all(a)); + assertEqualIntA(a, ARCHIVE_OK, + archive_read_open_memory(a, buff, used1)); + for (i = 0; i < 100; i++) { + sprintf(path, "file%03d", i); + if (!assertEqualInt(ARCHIVE_OK, + archive_read_next_header(a, &ae))) + break; + assertEqualString(path, archive_entry_pathname(ae)); + assertEqualInt((int)datasize, archive_entry_size(ae)); + } + assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a)); + } + assertEqualInt(ARCHIVE_OK, archive_read_free(a)); + + /* + * Repeat the cycle again, this time setting some compression + * options. + */ + assert((a = archive_write_new()) != NULL); + assertEqualIntA(a, ARCHIVE_OK, archive_write_set_format_ustar(a)); + assertEqualIntA(a, ARCHIVE_OK, + archive_write_set_bytes_per_block(a, 10)); + assertEqualIntA(a, ARCHIVE_OK, archive_write_add_filter_zstd(a)); + assertEqualIntA(a, ARCHIVE_FAILED, + archive_write_set_filter_option(a, NULL, "nonexistent-option", "0")); + assertEqualIntA(a, ARCHIVE_FAILED, + archive_write_set_filter_option(a, NULL, "compression-level", "abc")); + assertEqualIntA(a, ARCHIVE_FAILED, + archive_write_set_filter_option(a, NULL, "compression-level", "25")); /* too big */ + assertEqualIntA(a, ARCHIVE_OK, + archive_write_set_filter_option(a, NULL, "compression-level", "9")); + assertEqualIntA(a, ARCHIVE_OK, + archive_write_set_filter_option(a, NULL, "compression-level", "15")); + assertEqualIntA(a, ARCHIVE_OK, archive_write_open_memory(a, buff, buffsize, &used2)); + for (i = 0; i < 100; i++) { + sprintf(path, "file%03d", i); + assert((ae = archive_entry_new()) != NULL); + archive_entry_copy_pathname(ae, path); + archive_entry_set_size(ae, datasize); + archive_entry_set_filetype(ae, AE_IFREG); + assertEqualIntA(a, ARCHIVE_OK, archive_write_header(a, ae)); + assertA(datasize == (size_t)archive_write_data(a, data, datasize)); + archive_entry_free(ae); + } + assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a)); + assertEqualInt(ARCHIVE_OK, archive_write_free(a)); + + failure("compression-level=15 wrote %d bytes, default wrote %d bytes", + (int)used2, (int)used1); + assert(used2 < used1); + + assert((a = archive_read_new()) != NULL); + assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_all(a)); + r = archive_read_support_filter_zstd(a); + if (r == ARCHIVE_WARN) { + skipping("zstd reading not fully supported on this platform"); + } else { + assertEqualIntA(a, ARCHIVE_OK, + archive_read_support_filter_all(a)); + assertEqualIntA(a, ARCHIVE_OK, + archive_read_open_memory(a, buff, used2)); + for (i = 0; i < 100; i++) { + sprintf(path, "file%03d", i); + failure("Trying to read %s", path); + if (!assertEqualIntA(a, ARCHIVE_OK, + archive_read_next_header(a, &ae))) + break; + assertEqualString(path, archive_entry_pathname(ae)); + assertEqualInt((int)datasize, archive_entry_size(ae)); + } + assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a)); + } + assertEqualInt(ARCHIVE_OK, archive_read_free(a)); + + /* + * Test various premature shutdown scenarios to make sure we + * don't crash or leak memory. + */ + assert((a = archive_write_new()) != NULL); + assertEqualIntA(a, ARCHIVE_OK, archive_write_add_filter_zstd(a)); + assertEqualInt(ARCHIVE_OK, archive_write_free(a)); + + assert((a = archive_write_new()) != NULL); + assertEqualIntA(a, ARCHIVE_OK, archive_write_add_filter_zstd(a)); + assertEqualInt(ARCHIVE_OK, archive_write_close(a)); + assertEqualInt(ARCHIVE_OK, archive_write_free(a)); + + assert((a = archive_write_new()) != NULL); + assertEqualIntA(a, ARCHIVE_OK, archive_write_set_format_ustar(a)); + assertEqualIntA(a, ARCHIVE_OK, archive_write_add_filter_zstd(a)); + assertEqualInt(ARCHIVE_OK, archive_write_close(a)); + assertEqualInt(ARCHIVE_OK, archive_write_free(a)); + + assert((a = archive_write_new()) != NULL); + assertEqualIntA(a, ARCHIVE_OK, archive_write_set_format_ustar(a)); + assertEqualIntA(a, ARCHIVE_OK, archive_write_add_filter_zstd(a)); + assertEqualIntA(a, ARCHIVE_OK, archive_write_open_memory(a, buff, buffsize, &used2)); + assertEqualInt(ARCHIVE_OK, archive_write_close(a)); + assertEqualInt(ARCHIVE_OK, archive_write_free(a)); + + /* + * Clean up. + */ + free(data); + free(buff); +} diff --git a/contrib/libarchive/tar/bsdtar.1 b/contrib/libarchive/tar/bsdtar.1 index cdc317b6cf4d..132e1145794c 100644 --- a/contrib/libarchive/tar/bsdtar.1 +++ b/contrib/libarchive/tar/bsdtar.1 @@ -25,7 +25,7 @@ .\" .\" $FreeBSD$ .\" -.Dd February 25, 2017 +.Dd October 1, 2017 .Dt TAR 1 .Os .Sh NAME @@ -302,19 +302,18 @@ containing the string Compress the resulting archive with .Xr xz 1 . In extract or list modes, this option is ignored. -Note that, unlike other +Note that this .Nm tar -implementations, this implementation recognizes XZ compression -automatically when reading archives. +implementation recognizes XZ compression automatically when reading archives. .It Fl j , Fl Fl bzip , Fl Fl bzip2 , Fl Fl bunzip2 (c mode only) Compress the resulting archive with .Xr bzip2 1 . In extract or list modes, this option is ignored. -Note that, unlike other +Note that this .Nm tar -implementations, this implementation recognizes bzip2 compression -automatically when reading archives. +implementation recognizes bzip2 compression automatically when reading +archives. .It Fl k , Fl Fl keep-old-files (x mode only) Do not overwrite existing files. @@ -337,25 +336,41 @@ Issue a warning message unless all links to each file are archived. Compress the resulting archive with .Xr lrzip 1 . In extract or list modes, this option is ignored. +Note that this +.Nm tar +implementation recognizes lrzip compression automatically when reading +archives. .It Fl Fl lz4 (c mode only) Compress the archive with lz4-compatible compression before writing it. -In input mode, this option is ignored; lz4 compression is recognized -automatically on input. +In extract or list modes, this option is ignored. +Note that this +.Nm tar +implementation recognizes lz4 compression automatically when reading archives. +.It Fl Fl zstd +(c mode only) +Compress the archive with zstd-compatible compression before writing it. +In extract or list modes, this option is ignored. +Note that this +.Nm tar +implementation recognizes zstd compression automatically when reading archives. .It Fl Fl lzma (c mode only) Compress the resulting archive with the original LZMA algorithm. +In extract or list modes, this option is ignored. Use of this option is discouraged and new archives should be created with .Fl Fl xz instead. -Note that, unlike other +Note that this .Nm tar -implementations, this implementation recognizes LZMA compression -automatically when reading archives. +implementation recognizes LZMA compression automatically when reading archives. .It Fl Fl lzop (c mode only) Compress the resulting archive with .Xr lzop 1 . In extract or list modes, this option is ignored. +Note that this +.Nm tar +implementation recognizes LZO compression automatically when reading archives. .It Fl m , Fl Fl modification-time (x mode only) Do not extract modification time. @@ -577,6 +592,8 @@ A decimal integer from 4 to 7 specifying the lz4 compression block size .It Cm lz4:block-dependence Use the previous block of the block being compressed for a compression dictionary to improve compression ratio. +.It Cm zstd:compression-level +A decimal integer from 1 to 22 specifying the zstd compression level. .It Cm lzop:compression-level A decimal integer from 1 to 9 specifying the lzop compression level. .It Cm xz:compression-level @@ -826,28 +843,28 @@ is run in x mode as root. Compress the resulting archive with .Xr bzip2 1 . In extract or list modes, this option is ignored. -Note that, unlike other +Note that this .Nm tar -implementations, this implementation recognizes bzip2 compression -automatically when reading archives. +implementation recognizes bzip2 compression automatically when reading +archives. .It Fl Z , Fl Fl compress , Fl Fl uncompress (c mode only) Compress the resulting archive with .Xr compress 1 . In extract or list modes, this option is ignored. -Note that, unlike other +Note that this .Nm tar -implementations, this implementation recognizes compress compression -automatically when reading archives. +implementation recognizes compress compression automatically when reading +archives. .It Fl z , Fl Fl gunzip , Fl Fl gzip (c mode only) Compress the resulting archive with .Xr gzip 1 . In extract or list modes, this option is ignored. -Note that, unlike other +Note that this .Nm tar -implementations, this implementation recognizes gzip compression -automatically when reading archives. +implementation recognizes gzip compression automatically when reading +archives. .El .Sh ENVIRONMENT The following environment variables affect the execution of diff --git a/contrib/libarchive/tar/bsdtar.c b/contrib/libarchive/tar/bsdtar.c index d15214065472..e22e1d27285a 100644 --- a/contrib/libarchive/tar/bsdtar.c +++ b/contrib/libarchive/tar/bsdtar.c @@ -419,6 +419,7 @@ main(int argc, char **argv) case OPTION_LZIP: /* GNU tar beginning with 1.23 */ case OPTION_LZMA: /* GNU tar beginning with 1.20 */ case OPTION_LZOP: /* GNU tar beginning with 1.21 */ + case OPTION_ZSTD: if (compression != '\0') lafe_errc(1, 0, "Can't specify both -%c and -%c", opt, @@ -427,9 +428,10 @@ main(int argc, char **argv) switch (opt) { case OPTION_LRZIP: compression_name = "lrzip"; break; case OPTION_LZ4: compression_name = "lz4"; break; - case OPTION_LZIP: compression_name = "lzip"; break; - case OPTION_LZMA: compression_name = "lzma"; break; - case OPTION_LZOP: compression_name = "lzop"; break; + case OPTION_LZIP: compression_name = "lzip"; break; + case OPTION_LZMA: compression_name = "lzma"; break; + case OPTION_LZOP: compression_name = "lzop"; break; + case OPTION_ZSTD: compression_name = "zstd"; break; } break; case 'm': /* SUSv2 */ diff --git a/contrib/libarchive/tar/bsdtar.h b/contrib/libarchive/tar/bsdtar.h index 2bf42e8b28e7..612c2f67b273 100644 --- a/contrib/libarchive/tar/bsdtar.h +++ b/contrib/libarchive/tar/bsdtar.h @@ -181,7 +181,8 @@ enum { OPTION_USE_COMPRESS_PROGRAM, OPTION_UUENCODE, OPTION_VERSION, - OPTION_XATTRS + OPTION_XATTRS, + OPTION_ZSTD, }; int bsdtar_getopt(struct bsdtar *); diff --git a/contrib/libarchive/tar/cmdline.c b/contrib/libarchive/tar/cmdline.c index e36c545b3336..66cf4c2d196f 100644 --- a/contrib/libarchive/tar/cmdline.c +++ b/contrib/libarchive/tar/cmdline.c @@ -160,6 +160,7 @@ static const struct bsdtar_option { { "version", 0, OPTION_VERSION }, { "xattrs", 0, OPTION_XATTRS }, { "xz", 0, 'J' }, + { "zstd", 0, OPTION_ZSTD }, { NULL, 0, 0 } }; diff --git a/contrib/libarchive/tar/creation_set.c b/contrib/libarchive/tar/creation_set.c index 24cf3fcdd4ef..bdc607daeb20 100644 --- a/contrib/libarchive/tar/creation_set.c +++ b/contrib/libarchive/tar/creation_set.c @@ -80,9 +80,10 @@ get_filter_code(const char *suffix) { ".lzma", "lzma" }, { ".uu", "uuencode" }, { ".xz", "xz" }, + { ".zst", "zstd"}, { NULL, NULL } }; - + return get_suffix_code(filters, suffix); } @@ -121,6 +122,7 @@ decompose_alias(const char *suffix) { ".tzo", ".tar.lzo" }, { ".taZ", ".tar.Z" }, { ".tZ", ".tar.Z" }, + { ".tzst", ".tar.zst" }, { NULL, NULL } }; diff --git a/contrib/libarchive/tar/test/test_extract.tar.zst.uu b/contrib/libarchive/tar/test/test_extract.tar.zst.uu new file mode 100644 index 000000000000..3df0caf54758 --- /dev/null +++ b/contrib/libarchive/tar/test/test_extract.tar.zst.uu @@ -0,0 +1,6 @@ +begin 644 test_extract.tar.zst +M*+4O_010S0,`<L40$Z`5.(2U_RNV_[]L4V;Z_/R@1:7Y$3;9E`8$D$WI:W1) +M'58'D3->Y+>!0*5E/PM"$7^K^1VI3SS-AX&_W0KQWY!-Z1(`_4$%[$"]<T!A +L(*`#I!DXC4[J!6J8$!DJ$D"9$T*L]#G-$$/A`#`I`-(`UUKAU$Z@"`UXII`` +` +end diff --git a/contrib/libarchive/tar/test/test_extract_tar_zstd.c b/contrib/libarchive/tar/test/test_extract_tar_zstd.c new file mode 100644 index 000000000000..d67170ce3433 --- /dev/null +++ b/contrib/libarchive/tar/test/test_extract_tar_zstd.c @@ -0,0 +1,48 @@ +/*- + * Copyright (c) 2017 Sean Purcell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "test.h" +__FBSDID("$FreeBSD$"); + +DEFINE_TEST(test_extract_tar_zstd) +{ + const char *reffile = "test_extract.tar.zst"; + int f; + + extract_reference_file(reffile); + f = systemf("%s -tf %s >test.out 2>test.err", testprog, reffile); + if (f == 0 || canZstd()) { + assertEqualInt(0, systemf("%s -xf %s >test.out 2>test.err", + testprog, reffile)); + + assertFileExists("file1"); + assertTextFileContents("contents of file1.\n", "file1"); + assertFileExists("file2"); + assertTextFileContents("contents of file2.\n", "file2"); + assertEmptyFile("test.out"); + assertEmptyFile("test.err"); + } else { + skipping("It seems zstd is not supported on this platform"); + } +} diff --git a/contrib/libarchive/tar/test/test_option_acls.c b/contrib/libarchive/tar/test/test_option_acls.c index f2d51d661abf..325f319537af 100644 --- a/contrib/libarchive/tar/test/test_option_acls.c +++ b/contrib/libarchive/tar/test/test_option_acls.c @@ -483,7 +483,7 @@ DEFINE_TEST(test_option_acls) r = compare_acls("f", "acls_acls/f"); assertEqualInt(r, 1); - /* Extractl acls without acls */ + /* Extract acls without acls */ assertMakeDir("acls_noacls", 0755); clear_inheritance_flags("acls_noacls", acltype); r = systemf("%s -x -C acls_noacls -p --no-acls -f acls.tar >acls_noacls.out 2>acls_noacls.err", testprog); diff --git a/contrib/libarchive/tar/test/test_option_zstd.c b/contrib/libarchive/tar/test/test_option_zstd.c new file mode 100644 index 000000000000..73965e33f3af --- /dev/null +++ b/contrib/libarchive/tar/test/test_option_zstd.c @@ -0,0 +1,85 @@ +/*- + * Copyright (c) 2017 Sean Purcell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "test.h" +__FBSDID("$FreeBSD$"); + +DEFINE_TEST(test_option_zstd) +{ + char *p; + int r; + size_t s; + + /* Create a file. */ + assertMakeFile("f", 0644, "a"); + + /* Archive it with lz4 compression. */ + r = systemf("%s -cf - --zstd f >archive.out 2>archive.err", + testprog); + p = slurpfile(&s, "archive.err"); + p[s] = '\0'; + if (r != 0) { + if (strstr(p, "Unsupported compression") != NULL) { + skipping("This version of bsdtar was compiled " + "without zstd support"); + goto done; + } + /* POSIX permits different handling of the spawnp + * system call used to launch the subsidiary + * program: */ + /* Some systems fail immediately to spawn the new process. */ + if (strstr(p, "Can't launch") != NULL && !canZstd()) { + skipping("This version of bsdtar uses an external zstd program " + "but no such program is available on this system."); + goto done; + } + /* Some systems successfully spawn the new process, + * but fail to exec a program within that process. + * This results in failure at the first attempt to + * write. */ + if (strstr(p, "Can't write") != NULL && !canZstd()) { + skipping("This version of bsdtar uses an external zstd program " + "but no such program is available on this system."); + goto done; + } + /* On some systems the error won't be detected until closing + time, by a 127 exit error returned by waitpid. */ + if (strstr(p, "Error closing") != NULL && !canZstd()) { + skipping("This version of bsdcpio uses an external zstd program " + "but no such program is available on this system."); + return; + } + failure("--zstd option is broken: %s", p); + assertEqualInt(r, 0); + goto done; + } + free(p); + /* Check that the archive file has an lz4 signature. */ + p = slurpfile(&s, "archive.out"); + assert(s > 2); + assertEqualMem(p, "\x28\xb5\x2f\xfd", 4); + +done: + free(p); +} diff --git a/contrib/libarchive/test_utils/test_common.h b/contrib/libarchive/test_utils/test_common.h index 1425dd84b7df..dd7e41016d52 100644 --- a/contrib/libarchive/test_utils/test_common.h +++ b/contrib/libarchive/test_utils/test_common.h @@ -329,6 +329,9 @@ int canLrzip(void); /* Return true if this platform can run the "lz4" program. */ int canLz4(void); +/* Return true if this platform can run the "zstd" program. */ +int canZstd(void); + /* Return true if this platform can run the "lzip" program. */ int canLzip(void); diff --git a/contrib/libarchive/test_utils/test_main.c b/contrib/libarchive/test_utils/test_main.c index 0e1413693f4a..0e8cc9fc56a2 100644 --- a/contrib/libarchive/test_utils/test_main.c +++ b/contrib/libarchive/test_utils/test_main.c @@ -2319,6 +2319,21 @@ canLz4(void) } /* + * Can this platform run the zstd program? + */ +int +canZstd(void) +{ + static int tested = 0, value = 0; + if (!tested) { + tested = 1; + if (systemf("zstd -V %s", redirectArgs) == 0) + value = 1; + } + return (value); +} + +/* * Can this platform run the lzip program? */ int |